如何编辑此 python 脚本以确保所有文件路径也是唯一的？

Question

这是有问题的脚本 -

import os
import re

# Define the folder to start the search from
start_folder = "path"

# Define the name of the output file
output_file = "output.txt"

# Walk through all the subdirectories and files starting from the start folder
for root, dirs, files in os.walk(start_folder):

    # Loop through all the files in the current directory
    for file_name in files:

        # Check if the file is an HTML file
        if file_name.endswith(".html"):

            # Get the full path of the file
            file_path = os.path.join(root, file_name)

            # Open the file for reading
            with open(file_path, "r", encoding="UTF-8") as html_file:

                # Read the contents of the file into a string
                file_content = html_file.read()

                # Find all the http links in the file
                http_links = re.findall("(http[^<>'\" ]+)", file_content)

                # Find all the https links in the file
                https_links = re.findall("(https[^<>'\" ]+)", file_content)

                # Combine the http and https links into a single list
                all_links = http_links + https_links

                # Keep track of the unique links for this file
                unique_links = set()

                # Loop through all the links found in the file
                for link in all_links:

                    # Check if the link is already in the set of unique links for this file
                    if link not in unique_links:

                        # If the link is not in the set, add it to the set and write it to the output file
                        unique_links.add(link)
                        with open(output_file, "a", encoding="UTF-8") as f:
                            f.write(file_path + "\n-\n")
                            f.write(link + "\n")
                            f.write("\n")

# Print "Done scanning" when the script is finished
print("Done scanning")

如果我不希望代码太长，那么使文件路径也唯一的最佳方法是什么？我想改变格式从：

同路-

关联

同路-

下一个链接

对此：

同路-

关联

下一个链接

Answer 1

这应该可以解决问题：

改变这部分

unique_links = set()

# Loop through all the links found in the file
for link in all_links:

    # Check if the link is already in the set of unique links for this file
    if link not in unique_links:

        # If the link is not in the set, add it to the set and write it to the output file
        unique_links.add(link)
        with open(output_file, "a", encoding="UTF-8") as f:
            f.write(file_path + "
-
")
            f.write(link + "
")
            f.write("
")

为了这

unique_links = set(all_links)

#write the new links into the output file

with open(output_file, "a", encoding="UTF-8") as f:
    
    # identify the origin file
    f.write(file_path + "
-
") 
    
    # write each unique link found 
    for link in unique_links:
        f.write(link + "
")
        f.write("
")
            
      

Answer 2

import pathlib

ALL_PATHS = pathlib.Path("path/to/my/links")
# iterdir() method creates an iterator that lists the files randomly
all_links = list(ALL_PATHS.iterdir())
for link in all_links:
    with open(link) as f:
        f.write(...)