import{_ as s}from"./plugin-vue_export-helper-DlAUqK2U.js";import{c as n,o as e,a}from"./app-CQ4eKaQa.js";const l={},o=a(`

获取重复的文件

背景和环境说明

本程序使用 python 3.10+
机器学习，样本处理过程中会有重复的数据。这里的样本是图片，本程序根据文件MD5 获取重复文件。

依赖包下载

pip install -U NStudyPy

使用

from NStudyPy import PyFile

if __name__ == '__main__':
    PyFile.get_repeat_file(r'F:\\temp\\cards', is_recursive=True)

is_recursive 参数来限定是否递归子目录

核心源码

def get_repeat_file(path: str, is_recursive=True) -> dict:
    """
    获取重复文件
    :param path: 路径
    :param is_recursive:  是否递归
    :return: dict {"md5" : [file1,file2]}
    """
    file_dict = dict()
    for f in get_file_list(path, is_recursive):
        md5 = get_md5(f)
        if md5 not in file_dict:
            file_dict[md5] = [f]
        else:
            file_dict[md5].append(f)
    repeat_files = dict()
    for k, v in file_dict.items():
        if len(v) > 1:
            repeat_files.update({k: v})
            print(f'{k} {len(v)}')
    # return list(filter(lambda x: len(x) > 1, file_dict.values()))
    return repeat_files

get_file_list 获取文件 , 参加下篇随笔来介绍。
get_md5 获取文件MD5 , 参加下篇随笔来介绍。

`,11),p=[o];function t(c,r){return e(),n("div",null,p)}const D=s(l,[["render",t],["__file","3be333.html.vue"]]),y=JSON.parse(`{"path":"/p2024/01/03/3be333.html","title":"获取重复的文件","lang":"zh-CN","frontmatter":{"permalink":"/p2024/01/03/3be333.html","lang":"zh-CN","title":"获取重复的文件","description":"获取重复的文件","isOriginal":true,"date":"2024-06-03T00:00:00.000Z","category":["Python3"],"tag":["Python3"],"head":[["meta",{"name":"keywords","content":"获取重复的文件,Python3"}],["meta",{"property":"og:url","content":"https://nstudy.org/p2024/01/03/3be333.html"}],["meta",{"property":"og:site_name","content":"Jack's 新学习基地"}],["meta",{"property":"og:title","content":"获取重复的文件"}],["meta",{"property":"og:description","content":"获取重复的文件"}],["meta",{"property":"og:type","content":"article"}],["meta",{"property":"og:locale","content":"zh-CN"}],["meta",{"property":"og:updated_time","content":"2024-10-26T07:22:00.000Z"}],["meta",{"property":"article:author","content":"Jack"}],["meta",{"property":"article:tag","content":"Python3"}],["meta",{"property":"article:published_time","content":"2024-06-03T00:00:00.000Z"}],["meta",{"property":"article:modified_time","content":"2024-10-26T07:22:00.000Z"}],["script",{"type":"application/ld+json"},"{\\"@context\\":\\"https://schema.org\\",\\"@type\\":\\"Article\\",\\"headline\\":\\"获取重复的文件\\",\\"image\\":[\\"\\"],\\"datePublished\\":\\"2024-06-03T00:00:00.000Z\\",\\"dateModified\\":\\"2024-10-26T07:22:00.000Z\\",\\"author\\":[{\\"@type\\":\\"Person\\",\\"name\\":\\"Jack\\",\\"url\\":\\"https://nstudy.org\\"}]}"]],"sticky":true},"headers":[{"level":2,"title":"获取重复的文件","slug":"获取重复的文件","link":"#获取重复的文件","children":[]},{"level":2,"title":"背景和环境说明","slug":"背景和环境说明","link":"#背景和环境说明","children":[]},{"level":2,"title":"依赖包下载","slug":"依赖包下载","link":"#依赖包下载","children":[]},{"level":2,"title":"使用","slug":"使用","link":"#使用","children":[]},{"level":2,"title":"核心源码","slug":"核心源码","link":"#核心源码","children":[]}],"git":{"createdTime":1729927320000,"updatedTime":1729927320000,"contributors":[{"name":"lizhq","email":"lizhq08@163.com","commits":1}]},"readingTime":{"minutes":0.87,"words":260},"filePathRelative":"01.机器学习/03.工具/20240603_获取重复的文件.md","localizedDate":"2024年6月3日","excerpt":"

获取重复的文件

\\n

背景和环境说明

\\n
\\n

本程序使用 python 3.10+
\\n机器学习，样本处理过程中会有重复的数据。这里的样本是图片，本程序根据文件MD5 获取重复文件。

\\n
\\n

依赖包下载

\\n
\\n

pip install -U NStudyPy

\\n
\\n

使用

\\n
from NStudyPy import PyFile\\n\\nif __name__ == '__main__':\\n    PyFile.get_repeat_file(r'F:\\\\temp\\\\cards', is_recursive=True)
\\n
"}`);export{D as comp,y as data};