18 lines
484 B
Python
18 lines
484 B
Python
import os,sys
|
|
from multiprocessing import Process, Pool
|
|
|
|
ScriptLocation = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.append(ScriptLocation + "/lib")
|
|
|
|
import bulkCrawler
|
|
|
|
crawler = bulkCrawler.bulkCrawler(str(sys.argv[1]),str(sys.argv[2]))
|
|
crawler.writeHeader()
|
|
|
|
if __name__ == "__main__":
|
|
with Pool(processes=4) as pool:
|
|
pool.map(crawler.excution, range(len(crawler.crawlCmd())))
|
|
crawler.timeLog()
|
|
|
|
#for i in range(len(crawler.crawlCmd())):
|
|
# print(crawler.crawlCmd()[i]) |