【问题标题】:Python Fastest way to check and update Recursively file path if missing in DB如果在数据库中丢失,Python 检查和更新递归文件路径的最快方法
【发布时间】:2021-06-03 19:54:26
【问题描述】:

输入文件路径

示例:“/A/B/C/D/E/F”

  1. 如果不存在则检查路径“/A/B/C/D/E/F”是否存在

  2. 从“/A/B/C/D/E/F”中删除“/F”并检查system_file_paths中的“/A/B/C/D/E/”

    一个。如果存在,则创建 F 路径

    b.返回 F_ID

    c。如果“/A/B/C/D/E”不存在,则进行下一步

  3. 从“/A/B/C/D/E/F”中删除 /E/F 并检查 system_file_paths 中存在的“/A/B/C/D”

    一个。如果存在,创建 E/F

    b.返回 F_ID

    c。如果“/A/B/C/D”不存在,则进行下一步

  4. 从“/A/B/C/D/E/F”中删除 /D/E/F 并检查 system_file_paths 列表中存在的“/A/B/C”

    一个。如果存在,则创建 D/E/F

    b.返回 F_ID

    c。如果“/A/B/C”不存在,则进行下一步

做,直到最后一步A,如果不存在,

我已经编写了代码,它按预期工作,但是性能正在降级,因为我有大量文件,例如 1000 的millons 和 file_path 长度

注意:我不是要在操作系统中创建任何目录,而是在应用程序数据库中更新

我的代码:

# Random system generation input paths for testing 
import uuid
def file_path_list(num):
    file_path_list = []
    temp_list = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']
    for i in range(num):
        temp = "{0}".format(i)
        for j in temp_list:
            temp += "{0}".format(j)
        file_path_list.append(temp)

    temp_list = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h','1','2','3','4']
    for i in range(num):
        temp = "{0}".format(i)
        for j in temp_list:
            temp += "{0}".format(j)

        file_path_list.append(temp)

    temp_list = ['a', 'b', 'c', 'd', 'x', 'y', 'z']
    for i in range(num):
        temp = "{0}".format(i)
        for j in temp_list:
            temp += "{0}".format(j)
        file_path_list.append(temp)
    

    return file_path_list

path_trees = file_path_list(10)

system_file_paths = {}
for path in path_trees:
    system_file_paths[path] = str(uuid.uuid4())

# ************             system paths generation completed  for testing       ***************

# Input file 
file_path = 'A/B/C/D/E/F'

def update_db(path_name):
    _id = str(uuid.uuid4())
    system_file_paths[path_name] = _id
    return _id


# checking File path exist
def check_file_path_exists(file_path):
    if file_path in system_file_paths:
        return True
    return False

# Looking solution, to speed up process performance, by applying any suitable algorithm and reduces multiple checks, in this method
def process_to_update_db(file_path):
    # Split file path 
    file_path_split = file_path.split("/")
    # Iterate loop
    for reverse_path_indent in range(len(file_path_split) -1 , -1, -1):

        # If iterate until last value, create root path
        if reverse_path_indent == 0:
            if not check_file_path_exists(file_path_split[0]):
                _id = update_db(file_path_split[0])
            for path_indent in range(1, len(file_path_split)):
                file_path = "/".join(file_path_split[0: path_indent + 1])
                _id = update_db(file_path)
            return _id
            #_id = update_db(file_path_split[0])

        file_path_check = "/".join(file_path_split[0: reverse_path_indent])
        if not check_file_path_exists(file_path_check):
            continue
        else:
            for path_indent in range(reverse_path_indent, len(file_path_split)):
                file_path = "/".join(file_path_split[0: path_indent + 1])
                _id = update_db(file_path)
            return _id





if not check_file_path_exists(file_path):
    _id = process_to_update_db(file_path)
    print(system_file_paths)
    # Cross Verify path is created
    if check_file_path_exists(file_path):
        print(0)
    else:
        print(1)
    

【问题讨论】:

  • 不是 python,但可能有帮助:在 linux 中,命令 mkdir -p /A/B/C/D/E/F 完全符合您的要求。创建所有父目录(如果需要)。
  • 是的,但我没有使用操作系统,在应用程序中我们需要实现类似的东西
  • 您可以通过os.makedirs调用实现@MrSmith42的建议。

标签: python python-3.x algorithm sorting


【解决方案1】:

我又做了一些测试用例,并更正了我的代码:

# Random system generation input paths for testing 
import uuid, json

system_file_paths = {}

# ************             system paths generation completed  for testing       ***************

# Input file
file_path = [ 
    'A',
    'A/B/C/D/E/F',
    'A/B/C/D/G/H/I/J/K',
    'A/B/C/D/G/H/I/J/K/L/M/N/O/P/Q/R/S/T/U/V/W',
    'A/X/Y',
    'A/B/C/D/G/H/I/J/K/L/M/N/O/P/Q/R/S/T/U/V/W/Z',
    'A/B/C/D/G/H/I/J/A/B/C/D',
    'A/B/C/D/G/H/I/J/K/L/A/B/C/D/G/H/I/J/K/L/M/N/O/P/Q/R/S/T/U/V/W'
]

def update_db(path_name):
    global create_loop_count
    create_loop_count += 1
    _id = str(uuid.uuid4())
    system_file_paths[path_name] = _id
    return _id


# checking File path exist
def check_file_path_exists(file_path):
    if file_path in system_file_paths:
        try:
            return system_file_paths[file_path]
        except KeyError:
            return False
    return False

# Looking solution, to speed up process performance, by applying any suitable algorithm and reduces multiple checks, in this method
def process_to_update_db_original(file_path):
    global search_loop_count
    # Split file path 
    file_path_split = file_path.split("/")
    # Iterate loop
    for reverse_path_indent in range(len(file_path_split) -1 , -1, -1):
        search_loop_count += 1
        # If iterate until last value, create root path
        if reverse_path_indent == 0:
            if not check_file_path_exists(file_path_split[0]):
                _id = update_db(file_path_split[0])
            for path_indent in range(1, len(file_path_split)):
                file_path = "/".join(file_path_split[0: path_indent + 1])
                _id = update_db(file_path)
            return _id
            #_id = update_db(file_path_split[0])

        file_path_check = "/".join(file_path_split[0: reverse_path_indent])
        if not check_file_path_exists(file_path_check):
            continue
        else:
            for path_indent in range(reverse_path_indent, len(file_path_split)):
                file_path = "/".join(file_path_split[0: path_indent + 1])
                _id = update_db(file_path)
            return _id

# Looking solution, to speed up process performance, by applying any suitable algorithm and reduces multiple checks, in this method
def process_to_update_db_new(file_path):
    global search_loop_count
    # first of all, check to see if the path already exist:
    try:
        return system_file_paths[file_path]
    except KeyError:
        # do things the hard way
        # Split file path 
        if file_path:
            file_path_split = [file_path]
        if "/" in file_path:
            file_path_split = file_path.split("/")
        while '' in file_path_split:
            file_path_split.remove('')
        start_index = 0
        end_index = len(file_path_split) - 1
        current_path = ''
        build_path_start_index = 0
        # bias seach to the last directory minus one, assuming that most directory lists add a new path at the end
        current_index = end_index - 1

        # let's do a binary search on the paths to find where we need to start adding directories
        while start_index != end_index:
            search_loop_count += 1

            for i in range(build_path_start_index, current_index+1):
                if i != 0:
                    current_path += '/'
                current_path += file_path_split[i]

            #print("start_index: %d, end_index: %d, current_index: %d : %s"%(start_index,end_index,current_index,current_path))

            if check_file_path_exists(current_path):
                start_index = current_index+1
                if start_index > end_index:
                    start_index = end_index
                build_path_start_index = start_index
            else:
                current_path = ''
                build_path_start_index = 0
                end_index = current_index-1
                if end_index < start_index:
                    end_index = start_index

            current_index = int((end_index-start_index)/2) + start_index

        # build up the existing path
        for i in range(build_path_start_index, start_index+1):
            if i != 0:
                current_path += '/'
            current_path += file_path_split[i]

        # do the actual insertion of the UUID for new paths
        for i in range(start_index, len(file_path_split)):
            if i == start_index:
                # check if current path exists
                if check_file_path_exists(current_path):
                    # move on to next index
                    continue
                else:
                    # current_path doesn't exist, create it below
                    pass
            else:
                # append next path item
                if i != 0:
                    current_path += '/'
                current_path += file_path_split[i]

            current_id = update_db(current_path)

        return current_id


for current_path in file_path:
    if not check_file_path_exists(current_path):
        search_loop_count = 0
        create_loop_count = 0
        _id = process_to_update_db_new(current_path)
        # Cross Verify path is created
        if check_file_path_exists(current_path):
            print("SUCCESS")
        else:
            print("!!!FAILURE!!!")
        print("search_loop_count: %d create_loop_count %d"%(search_loop_count, create_loop_count))

print(json.dumps(system_file_paths,indent=4,sort_keys=True))


create_loop_count你的和我的一样,但是搜索循环的数量不同:

            | search_loop_count |
------------|-------------------|
Test String | Original |  New   |
------------|----------|--------|
    1       |    1     |   0    |
    2       |    5     |   2    |
    3       |    5     |   3    |
    4       |    12    |   5    |
    5       |    2     |   1    |
    6       |    1     |   1    |
    7       |    4     |   4    |
    8       |    21    |   5    |

如果路径长度为数千,正如您在问题中所说,那么差异会很大。

【讨论】:

    猜你喜欢
    • 1970-01-01
    • 2015-12-21
    • 1970-01-01
    • 1970-01-01
    • 2018-11-29
    • 2011-08-04
    • 2015-03-11
    • 1970-01-01
    • 2014-08-09
    相关资源
    最近更新 更多