Sum of values in a function called from thread
I have few text files containing some numbers in each line. I read the file and displayed total of each number in the function.
from Queue import Queue from threading import Thread import os enclosure_queue = Queue() list= def getAllFiles(): for root, dirs, files in os.walk("C:/Users/test"): for file in files: if file.endswith(".txt"): file_path= os.path.join(root, file) list.append(file_path) def calc(i, q): while True: file = q.get() fileData = open(file, 'r') add=0 for line in fileData: add= add + int(line) print str(add) + '\n' q.task_done() getAllFiles() num_fetch_threads = len(list) for i in range(num_fetch_threads): worker = Thread(target=calc, args=(i, enclosure_queue,)) worker.setDaemon(True) worker.start() for ind_file in list: enclosure_queue.put(ind_file) enclosure_queue.join()
It displays the sum of data in lines of individual file, but I need to add up the results. For example if the calc function's add has 300 , 200 and 500 , I want final result as 1000 . I thought of adding each result to a list and then splitting it in another function and adding them. Is there any better solution ?
No need to use Queue here. Use multiprocessing.Pool.map, and change your calc method accordingly. Also threading.Thread does not return results, whereas multiprocessing.Pool.map returns results.
import multiprocessing import os def getAllFiles(): my_files = list() for root, dirs, files in os.walk("C:/Users/test"): for file in files: if file.endswith(".txt"): file_path= os.path.join(root, file) my_files.append(file_path) return my_files def calc(file): with open(file, 'r') as f: return sum(map(int, f.readlines())) if __name__ == '__main__': my_files = getAllFiles() num_fetch_threads = len(my_files) pool = multiprocessing.Pool(processes=num_fetch_threads) results = pool.map(calc, my_files) result = sum(results) print result