如果您正在处理大型数据集并希望探索并行化，您可以考虑使用Python中的concurrent.futures模块进行多处理。这样，每个进程可以同时处理CSV文件子集的阅读和处理。

import os
import pandas as pd
from concurrent.futures import ProcessPoolExecutor

def process_file(row):
    folder_name = row['File ID'].strip()
    file_name = row['File Name'].strip()
    file_path = os.path.join(root_path, folder_name, file_name)

    file_data = pd.read_csv(file_path, names=['Case', f'{folder_name}_{file_name}'], sep='\t')
    file_data_transposed = file_data.set_index('Case').T.reset_index(drop=True)
    file_data_transposed.insert(loc=0, column='folder_file_id', value=str(folder_name+'_'+file_name))

    return file_data_transposed

root_path = r'G:\path'

# Adjust the number of processes based on your system's capabilities
num_processes = 4  # You can experiment with different values

with ProcessPoolExecutor(max_workers=num_processes) as executor:
    results = list(executor.map(process_file, df.itertuples(index=False)))

merged_data = pd.concat(results, axis=0, ignore_index=True)

字符串

代码慢的原因是因为你在循环中使用了concat。你应该在python字典中收集数据，然后在最后执行一个concat。
只有很少的改进：

import pathlib
import pandas as pd

root_path = pathlib.Path('root')  # use pathlib instead of os.path

data = {}
# use enumerate rather than create an external counter
for count, (_, row) in enumerate(df.iterrows(), 1):
    folder_name = row['File ID'].strip()
    file_name = row['File Name'].strip()
    file_path = root_path / folder_name / file_name
    folder_file_id = f'{folder_name}_{file_name}'

    file_data = pd.read_csv(file_path, header=None, sep='\t',
                            names=['Case', folder_file_id],
                            memory_map=True, low_memory=False)
    data[folder_file_id] = file_data.set_index('Case').squeeze()
    print(count)

merged_data = (pd.concat(data, names=['folder_file_id'])
                 .unstack('Case').reset_index())

字符串
输出量：

>>> merged_data
Case       folder_file_id       0       1       2       3       4
0     folderA_file001.txt  1234.0  5678.0  9012.0  3456.0  7890.0
1     folderB_file002.txt  4567.0  8901.0  2345.0  6789.0     NaN

型
输入数据：

>>> df
   File ID    File Name
0  folderA  file001.txt
1  folderB  file002.txt

>>> cat root/folderA/file001.txt
0   1234
1   5678
2   9012
3   3456
4   7890

>>> cat root/folderB/file002.txt
0   4567
1   8901
2   2345
3   6789

型
多线程版本：

from concurrent.futures import ThreadPoolExecutor
import pathlib
import pandas as pd

root_path = pathlib.Path('root')

def read_csv(args):
    count, row = args  # expand arguments
    folder_name = row['File ID'].strip()
    file_name = row['File Name'].strip()
    file_path = root_path / folder_name / file_name
    folder_file_id = f'{folder_name}_{file_name}'

    file_data = pd.read_csv(file_path, header=None, sep='\t',
                            names=['Case', folder_file_id],
                            memory_map=True, low_memory=False)
    print(count)
    return folder_file_id, file_data.set_index('Case').squeeze()

with ThreadPoolExecutor(max_workers=2) as executor:
    batch = enumerate(df[['File ID', 'File Name']].to_dict('records'), 1)
    data = executor.map(read_csv, batch)

merged_data = (pd.concat(dict(data), names=['folder_file_id'])
                 .unstack('Case').reset_index())

型

检查这个代码。

import pandas as pd
import os

merged_data_list = []

for index, row in df.iterrows():
    folder_name = row['File ID'].strip()
    file_name = row['File Name'].strip()
    file_path = os.path.join(root_path, folder_name, file_name)
    
    file_data = pd.read_csv(file_path, names=['Case', f'{folder_name}_{file_name}'], sep='\t')
    file_data_transposed = file_data.set_index('Case').T.reset_index(drop=True)
    file_data_transposed.insert(loc=0, column='folder_file_id', value=str(folder_name+'_'+file_name))
    
    merged_data_list.append(file_data_transposed)

# Concatenate all DataFrames in the list at once
merged_data = pd.concat(merged_data_list, axis=0, ignore_index=True)

字符串