Sum of values in a function called from thread

I have few text files containing some numbers in each line. I read the file and displayed total of each number in the function.

from Queue import Queue
from threading import Thread
import os

enclosure_queue = Queue()
list=[]

def getAllFiles():
    for root, dirs, files in os.walk("C:/Users/test"):
        for file in files:
            if file.endswith(".txt"):
                file_path= os.path.join(root, file) 
                list.append(file_path)

def calc(i, q):
    while True:
        file = q.get()
        fileData = open(file, 'r')
        add=0
        for line in fileData:
            add= add + int(line)    
        print str(add) + '\n'
        q.task_done()

getAllFiles()
num_fetch_threads = len(list)
for i in range(num_fetch_threads):
    worker = Thread(target=calc, args=(i, enclosure_queue,))
    worker.setDaemon(True)
    worker.start()

for ind_file in list:
    enclosure_queue.put(ind_file)

enclosure_queue.join()

It displays the sum of data in lines of individual file, but I need to add up the results. For example if the calc function's add has 300 , 200 and 500 , I want final result as 1000 . I thought of adding each result to a list and then splitting it in another function and adding them. Is there any better solution ?

Answers


No need to use Queue here. Use multiprocessing.Pool.map, and change your calc method accordingly. Also threading.Thread does not return results, whereas multiprocessing.Pool.map returns results.

import multiprocessing
import os

def getAllFiles():
    my_files = list()
    for root, dirs, files in os.walk("C:/Users/test"):
        for file in files:
            if file.endswith(".txt"):
                file_path= os.path.join(root, file) 
                my_files.append(file_path)
    return my_files

def calc(file):
    with open(file, 'r') as f:
        return sum(map(int, f.readlines()))

if __name__ == '__main__':
    my_files = getAllFiles()
    num_fetch_threads = len(my_files)
    pool = multiprocessing.Pool(processes=num_fetch_threads)
    results = pool.map(calc, my_files)
    result = sum(results)
    print result

Need Your Help

gnuplot-iostream won't compile

c++ boost gnuplot

I was wondering if someone could help me with this.