Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/fmultiprocess.py : 38%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/local/bin/python
2# encoding: utf-8
3"""
4*A function to quickly add multiprocessing to any program*
6:Author:
7 David Young
9:Date Created:
10 November 9, 2017
11"""
12from __future__ import division
13################# GLOBAL IMPORTS ####################
14from past.utils import old_div
15import sys
16import os
17os.environ['TERM'] = 'vt100'
18from fundamentals import tools
19from multiprocess import cpu_count, Pool
20from functools import partial
21import inspect
22import psutil
25def fmultiprocess(
26 log,
27 function,
28 inputArray,
29 poolSize=False,
30 timeout=3600,
31 **kwargs):
32 """multiprocess pool
34 **Key Arguments:**
35 - ``log`` -- logger
36 - ``function`` -- the function to multiprocess
37 - ``inputArray`` -- the array to be iterated over
38 - ``poolSize`` -- limit the number of CPU that are used in multiprocess job
39 - ``timeout`` -- time in sec after which to raise a timeout error if the processes have not completed
41 **Return:**
42 - ``resultArray`` -- the array of results
44 **Usage:**
46 .. code-block:: python
48 from fundamentals import multiprocess
49 # DEFINE AN INPUT ARRAY
50 inputArray = range(10000)
51 results = multiprocess(log=log, function=functionName, poolSize=10, timeout=300,
52 inputArray=inputArray, otherFunctionKeyword="cheese")
53 """
54 log.debug('starting the ``multiprocess`` function')
56 # DEFINTE POOL SIZE - NUMBER OF CPU CORES TO USE (BEST = ALL - 1)
57 if not poolSize:
58 poolSize = psutil.cpu_count()
60 if poolSize:
61 p = Pool(processes=poolSize)
62 else:
63 p = Pool()
65 cpuCount = psutil.cpu_count()
66 chunksize = int(old_div((len(inputArray) + 1), (cpuCount * 3)))
68 if chunksize == 0:
69 chunksize = 1
71 # chunksize = 1
73 # MAP-REDUCE THE WORK OVER MULTIPLE CPU CORES
74 if "log" in inspect.getargspec(function)[0]:
75 mapfunc = partial(function, log=log, **kwargs)
76 resultArray = p.map_async(mapfunc, inputArray, chunksize=chunksize)
77 else:
78 mapfunc = partial(function, **kwargs)
79 resultArray = p.map_async(mapfunc, inputArray, chunksize=chunksize)
81 resultArray = resultArray.get(timeout=timeout)
83 p.close()
84 p.join()
85 # p.terminate()
87 log.debug('completed the ``multiprocess`` function')
88 return resultArray