python_apps/crwlers/Sales.py

#!/usr/bin/env python
# -*- coding: UTF-8 -*-# enable debugging
import requests, bs4, urllib, sys, re, math, logging, pymysql, json
from urllib import parse
from datetime import datetime, timedelta

def getDate():
    if len(sys.argv) != 1:
        date = sys.argv[1]
        date = datetime.strptime(date,'%Y%m%d%H')
    else:
        date = datetime.now() - timedelta(minutes=5)
    if date.year%4 != 0:
        monRef = [31,28,31,30,31,30,31,31,30,31,30,31]
    else:
        monRef = [31,29,31,30,31,30,31,31,30,31,30,31]
    weekno = date.weekday() + 1
    if weekno == 7:
        weekno = 0
    res = {}
    res['hourlyStartdate'] = date.strftime('%Y%m%d%H') + "0000"
    res['hourlyEnddate'] = date.strftime('%Y%m%d%H') + "5959"
    res['hourlyDataName'] = date.strftime('%Y%m%d%H')
    res['DailyStartdate'] = date.strftime('%Y%m%d') + "000000"
    res['DailyEnddate'] = date.strftime('%Y%m%d') + "235959"
    res['DailyDataName'] = date.strftime('%Y%m%d')
    weekStart = date - timedelta(days=weekno)
    weekEnd = weekStart + timedelta(days=6)
    res['WeeklyStartdate'] = weekStart.strftime('%Y%m%d') + "000000"
    res['WeeklyEnddate'] = weekEnd.strftime('%Y%m%d') + "235959"
    res['WeeklyDataName'] = weekStart.strftime('%Y%m%d')
    res['MonthlyStartdate'] = date.strftime('%Y%m') + "01000000"
    res['MonthlyEnddate'] = date.strftime('%Y%m') + str(monRef[date.month-1]) + "235959"
    res['MonthlyDataName'] = date.strftime('%Y%m')
    return res

def getResults(startTime,endTime):
    url = "http://ibas.imarket.co.kr/sqlViewer/upsale/queryDataModReal.jsp"
    data = {
        'sdatereal': startTime,
        'edatereal': endTime,
        'depthreal': 1,
        'gu_1': 'gu_1',
        'gu_2': 'gu_2',
        'gu_3': 'gu_3',
        'gu_4': 'gu_4',
        'searchgu': 20
    }
    resp = requests.get(url, params = data)
    resp.raise_for_status()
    resp.encoding='UTF-8'
    jsonData = json.loads(resp.content)
    return jsonData

def getJSONRES():
    timeVar = getDate()
    tmpAr = ['hourly','Daily','Weekly','Monthly']
    res=[]
    for i in range(len(tmpAr)):
        rest = {}
        s = timeVar[tmpAr[i] + "Startdate"]
        e = timeVar[tmpAr[i] + "Enddate"]
        d = timeVar[tmpAr[i] + "DataName"]
        add_dict = { "DbDate": d, "type": tmpAr[i] }
        rest[tmpAr[i]] = getResults(s,e)
        for j in range(len(rest[tmpAr[i]])):
            rest[tmpAr[i]][j].update(add_dict)
        res.append(rest)
    return res

def nameNormal(text):
    text = text.replace(" ","")
    text = text.replace("ㄴ","")
    text = text.replace(" ","")
    return text

def dbconnect():
    conn = pymysql.connect(host='localhost', user='fin', password='mad(#lin',db='fin', unix_socket='/var/run/mysqld/mysqld.sock', charset='utf8')
    return conn

def cloumnMapping(dict) :
    res = {}
    res['date'] = dict['DbDate']
    res['category'] = nameNormal(dict['CLASSNM'])
    res['finalRevenue'] = dict['TOT_CAL_AMT']
    res['totalSales'] = dict['CAL_AMT']
    res['allSales'] = dict['AMT']
    res['claimSales'] = dict['CLM_AMT']
    res['finalDiscount'] = dict['CAL_DC_AMT']
    res['allDiscount'] = dict['DC_AMT']
    res['claimDiscount'] = dict['CLM_DC_AMT']
    res['finalCost'] = dict['TOT_CAL_MGT_AMT']
    res['totalCost'] = dict['MGT_AMT']
    res['claimCost'] = dict['CLM_MGT_AMT']
    res['profit'] = dict['MRG_AMT']
    res['profitRate'] = dict['MRG_RATE']
    res['mtsRevenue'] = dict['MTS_TOT_CAL_AMT']
    res['mtsSales'] = dict['MTS_CAL_AMT']
    res['mtsTotalSales'] = dict['MTS_AMT']
    res['mtsClaimSales'] = dict['CLM_MTS_AMT']
    res['mtsRevRate'] = dict['TOT_MST_RATE']
    res['mtsFinalDiscount'] = dict['MTS_CAL_DC_AMT']
    res['mtsTotalDiscount'] = dict['MTS_DC_AMT']
    res['mtsClaimDiscount'] = dict['CLM_MTS_DC_AMT']
    res['mtsFinalCost'] = dict['MTS_CAL_MGT_AMT']
    res['mtsTotalCost'] = dict['MTS_MGT_AMT']
    res['mtsClaimCost'] = dict['CLM_MTS_MGT_AMT']
    res['mtsProfit'] = dict['MTS_MRG_AMT']
    res['mtsProfitRate'] = dict['MTS_MRG_RATE']
    res['mtsQuantitySold'] = dict['MTS_ORD_CNT']
    res['mtsSoldSKU'] = dict['MTS_PRD_CNT']
    res['mtsClaimSKU'] = dict['MTS_CLM_PRD_CNT']
    res['quantitySold'] = dict['ORD_CNT']
    res['orders'] = dict['PO_CNT']
    res['claimOrders'] = dict['PO_CLM_CNT']
    res['GR'] = dict['GR_CNT']
    res['claimGR'] = dict['CLM_GR_CNT']
    res['soldSKU'] = dict['PRD_CNT']
    res['claimSKU'] = dict['CLM_PRD_CNT']
    res['customers'] = dict['PO_MEM_CNT']
    res['claimCustomers'] = dict['PO_CLM_MEM_CNT']
    return res

def checkExist(dict):
    conn = dbconnect()
    sql = "SELECT COUNT(*) FROM `" + dict["type"] + "Sales" + "` WHERE `date` = \"" + dict["DbDate"] + "\" AND `category` = \"" + nameNormal(dict["CLASSNM"]) + "\""
    try:
        with conn.cursor() as cursor:
            cursor.execute(sql)
            result = cursor.fetchone()
        conn.commit()
    finally:
        conn.close()
    return result[0]
    
def insertData(dict):
    conn = dbconnect()
    tablename = dict["type"] + "Sales"
    dictData = cloumnMapping(dict)
    dictData['no'] = ''
    cols = dictData.keys()
    vals = dictData.values()
    sql = "INSERT INTO " + tablename + " ("
    for cols in cols:
        sql += cols + ","
    sql = sql[:-1] + ") VALUES ("
    for vals in vals:
        sql += "\"" + vals + "\","
    sql = sql[:-1] + ")"
    try:
        with conn.cursor() as cursor:
            cursor.execute(sql)
        conn.commit()
    finally:
        conn.close()
    return sql

def updateData(dict):
    conn = dbconnect()
    tablename = dict["type"] + "Sales"
    dictData = cloumnMapping(dict)
    cols = dictData.keys()
    vals = dictData.values()
    sql = "UPDATE " + tablename + " SET "
    for k,v in dictData.items():
        sql += "%s='%s'," %(k,v)
    sql = sql[:-1]
    sql = sql + " WHERE `date` = \"" + dict["DbDate"] + "\" AND `category` = \"" + nameNormal(dict["CLASSNM"]) + "\""
    try:
        with conn.cursor() as cursor:
            cursor.execute(sql)
        conn.commit()
    finally:
        conn.close()
    return sql
    
    #curs = conn.cursor()

res = getJSONRES()

for i in range(len(res)):
    data1 = res[i]
    for tmp in data1:
        data2 = data1[tmp]        
        for j in range(len(data2)):
            data = data2[j]
            count = checkExist(data)
            if count == 0:
                insertData(data)
            else:
                updateData(data)