'''
Created on Jan 7, 2016

Generate assignment of invoices to be sent to Wells Fargo in csv format.
This is mainly called from the send_wf.py cgi script running on the Apache instance on our test server.

'WF' in comments = Wells Fargo

@todo: Properly truncate invoice amount value to fit WF's decimal(12,2) field size. TOTAL and DETAIL records!

@author: Wes
'''
import paramiko, csv, sys, logging, re, time, os, smtplib, sqlite3, StringIO, subprocess
from datetime import datetime

from sqlalchemy import create_engine, func, Table, text, Column
from sqlalchemy.types import VARCHAR
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy.sql.expression import update

from alert import send_alert_html

TEST_MODE = False #When 'True' files will be uploaded to the 'testin' directory on the remote server

''' CLIENT_CODE = 5867 #Old Sunbury Textile Mills client code. Last Used: 09/29/2017 '''
CLIENT_CODE = 4143 #New code assigned to us by WF. Effective: 10/02/2017
REMOTE_HOST = 'sftp.spscommerce.net'
REMOTE_PORT = 10022
MESSAGE = None
VALID_TRADE_STYLES = ['888888', '88888800', '88888803', '88888805', '88888809']

'''
The only valid trade style numbers at the time this pattern was created are:
888888
88888800
88888803
88888805
88888809 
'''
TRADE_STYLE_PATTERN = "888888(?P<tradeStyle>\d{0}$|0[0359]$)"
''' EXACTLY 10 digits '''
PHONE_NUM_PATTERN = "^\d{10}$"

#Moved connection string into sqlite db
for row in sqlite3.connect('D:\\FTP\\ftp.db').execute("SELECT * FROM connection_strings WHERE code = 'wfa'"):
    engine = create_engine(row[1], echo=False)

Base = declarative_base(engine)
metadata = Base.metadata

logger = None
LOG_LEVEL = logging.DEBUG
TERMS_CODES = { '00': 'No Terms' }
assignmentNumber = 0
fileNames = []
backouts = ['9999999999']

#Customer Master
class CustomerMaster(Base):
    __tablename__ = 'TXUYKD00'
    __table_args__ = {'autoload':True}
#Tables and Codes
class TXMYTX00(Base):
    __tablename__ = 'TXMYTX00'
    __table_args__ = {'autoload':True}
#Invoice header table
class InvoiceHeader(Base):
    __tablename__ = 'TXUYRE00'
    __table_args__ = {'autoload':True}
#Invoice detail table
class InvoiceDetail(Base):
    __tablename__ = 'TXUYRE01'
    __table_args__ = {'autoload':True}
#Order Header
class OrderHeader(Base):
    __tablename__ = 'TXUYUV00'
    __table_args__ = {'autoload':True}
#Backouts
class Invoice_Backouts(Base):
    __table__ = Table('Invoice_Backouts', Base.metadata,
        Column('b_Invoice', VARCHAR, primary_key=True),
        autoload=True, autoload_with=engine
        )

'''
Storing FTP site information in a sqlite db now to keep it out of source code
'''
def load_ftp_credentials():
    vault = sqlite3.connect('D:\\FTP\\ftp.db')
    cursor = vault.execute("SELECT *  from ftp_auth where code='wf'")
    for row in cursor:
        return row

'''
Upload files to WF's ftp site for processing.
Global var TEST_MODE dictates which remote dir the files are uploaded into.

files should contain a list of fully qualified file paths to be sent.
'''
def sendFiles(files):
    return
#    global logger
#    global TEST_MODE
#    if(TEST_MODE):
#        remoteDir = '/testin/'
#    else:
#        remoteDir = '/in/'
#
#    creds = load_ftp_credentials()
#    
#    logger.info('Opening SFTP connection to %s:%s',REMOTE_HOST,REMOTE_PORT)
#    transport = paramiko.Transport((REMOTE_HOST, REMOTE_PORT))
#    transport.connect(username = creds[3], password = creds[4])
#    sftp = paramiko.SFTPClient.from_transport(transport)
#    
#    for f in files:
#        path = remoteDir + f
#        logger.info('sftp.put(\'%s\', \'%s\')', ('D:\\FTP\\SEND_WF\\' + f), path)
#        sftp.put(('D:\\FTP\\SEND_WF\\' + f), path)
#        
#    logger.debug('DIR %s = %s', remoteDir, sftp.listdir(remoteDir[:-1]))
#    
#    logger.info('Closing SFTP connection.')
#    sftp.close()
#    transport.close()

'''
Opens connection to database returning a session object.
'''
def loadSession():
    global engine
    Session = sessionmaker(bind=engine)
    session = Session()
    return session

def initializeLogger(name='log'):
    global logger
    global MESSAGE
    logger = logging.getLogger(__name__)
    logger.setLevel(LOG_LEVEL)
    handler = logging.FileHandler('logs\\' + name + '.log')
    handler.setLevel(LOG_LEVEL)
    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    
    #Also send logs > DEBUG to stdout
    ch = logging.StreamHandler(sys.stdout)
    ch.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    #Capture log messages in a string for e-mail alert
    MESSAGE = StringIO.StringIO()
    sh = logging.StreamHandler(MESSAGE)
    sh.setLevel(logging.INFO)
    sh.setFormatter(formatter)
    logger.addHandler(sh)

'''
The result of this function is a single assignment file.
An assignment containing all of the invoices (or credits) for a single tradestyle on a given date.

Invoices and Credits (assuming all is done through the ERP method in jomar) are stored in the same
table.
An 'Invoice' for the purposes of transmitting to wf is defined as any invoice in the table with a
_positive_ dollar amount. 
A 'Credit' is similarly defined as any invoice with a _negative_ dolar amount.
There is a flag cloumn in the table that designates what type of invoice it is (SO, SR, etc.) however
Caleb has expressed that the +/- methodology works well.

                                 Valid Tradestyles:     00,       03,       05,       09
These are mapped to these factor accounts in Jomar: 888888, 88888803, 88888805, 88888809

d - the target date in YYYmmdd format
tStyle - The associated factor account for the target tradestyle ex. 888888 for 00
creditMemo - True to process credits, False to process invoices

ex: createAssignment(20160520, '888888', False) # Generate 00 invoice assignment for 05/20/16

This method returns a list containing the invoice/credit numbers that were processed. This is
used by the main method to build a complete list of invoices to flag as processed if the CSV files
are sent to Wells Fargo. This prevents any invoices processed by this script from being "backed out"
after the fact.
'''
def createAssignment(d, tStyle, creditMemo):
    global assignmentNumber
    global fileNames
    global backouts
    invoices = []
    #Check for invalid trade styles, The script ignores invalid trade styles and proceeds if it can.
    test = re.match(TRADE_STYLE_PATTERN, tStyle, flags=0)            
    if(test == None):
        if not(tStyle == ''):
            logger.warning('Ignoring Invalid Trade Style: \'%s\'', tStyle)
        return invoices
    elif(test.group('tradeStyle') == ''):
        tradeStyleCode = '01'
        logger.debug('Jomar trade style = %s --> Wells Fargo trade style = %s', tStyle, tradeStyleCode)
    else:
        tradeStyleCode = test.group('tradeStyle')
        logger.debug('Jomar trade style = %s --> Wells Fargo trade style = %s', tStyle, tradeStyleCode)
    
    #Get number of invoices for this trade style
    #logger.debug('SELECT COUNT(RRNR) FROM TXUYRE00 WHERE RDATLS = \'%s\' AND RFCT LIKE \'%s\'', d, tStyle)

    ###  GR PATCHES  ###
    if(creditMemo):
        logger.debug('SELECT COUNT(RRNR) FROM TXUYRE00 WHERE RDATLS = \'%s\' AND RFCT LIKE \'%s\' AND RWTF < 0;', d, tStyle)
        invoiceCount = session.query(InvoiceHeader.RRNR).filter(InvoiceHeader.RDATLS == d, InvoiceHeader.RFCT.in_(VALID_TRADE_STYLES), InvoiceHeader.RWTF < 0, ~InvoiceHeader.RRNR.in_(backouts)).count()
    else:
        logger.debug('SELECT COUNT(RRNR) FROM TXUYRE00 WHERE RDATLS = \'%s\' AND RFCT LIKE \'%s\' AND RWTF > 0;', d, tStyle)
        invoiceCount = session.query(InvoiceHeader.RRNR).filter(InvoiceHeader.RDATLS == d, InvoiceHeader.RFCT.in_(VALID_TRADE_STYLES), InvoiceHeader.RWTF > 0, ~InvoiceHeader.RRNR.in_(backouts)).count()
    ###  GR PATCHES  ###
        
    if(invoiceCount == 0):
        return invoices
    if(creditMemo):
        logger.info('**************************************************')
        logger.info('Total number of %s CREDITS: %s', tradeStyleCode, invoiceCount)
    else:
        logger.info('**************************************************')
        logger.info('Total number of %s INVOICES: %s', tradeStyleCode, invoiceCount)
    
    #Trade style is valid, increment assignment
    assignmentNumber += 1
        
    #Open file for writing
    fname = str(CLIENT_CODE) + tradeStyleCode + '_INV_' + str(d) + '_' + str(assignmentNumber) + '.csv'
    logger.debug('Creating %s. Using \'excel\' dialect for csv writer.', fname)
    f = open('SEND_WF\\' + fname, 'wb')
    writer = csv.writer(f, dialect='excel')
    fileNames.append(fname)
    
    #Get total dollar amount of all invoices for this trade style
    invoiceTotal = 0
    #logger.debug('SELECT SUM(RWTS) sum FROM TXUYRE00 WHERE RDATLS = \'%s\' AND RFCT LIKE \'%s\'', d, tStyle)

    ###  GR PATCHES  ###
    if(creditMemo):
        logger.debug('SELECT SUM(RWTS) sum FROM TXUYRE00 WHERE RDATLS = \'%s\' AND RFCT LIKE \'%s\' AND RWTF < 0;', d, tStyle)
        for row in session.query(func.sum(InvoiceHeader.RWTS).label("sum")).filter(InvoiceHeader.RDATLS == d, InvoiceHeader.RFCT.in_(VALID_TRADE_STYLES), InvoiceHeader.RWTF < 0, ~InvoiceHeader.RRNR.in_(backouts)):
            invoiceTotal = row.sum
    else:
        logger.debug('SELECT SUM(RWTS) sum FROM TXUYRE00 WHERE RDATLS = \'%s\' AND RFCT LIKE \'%s\' AND RWTF > 0;', d, tStyle)
        for row in session.query(func.sum(InvoiceHeader.RWTS).label("sum")).filter(InvoiceHeader.RDATLS == d, InvoiceHeader.RFCT.in_(VALID_TRADE_STYLES), InvoiceHeader.RWTF > 0, ~InvoiceHeader.RRNR.in_(backouts)):
            invoiceTotal = row.sum
    ###  GR PATCHES  ###
    
    if(creditMemo):
        logger.info('Total dollar amount for %s CREDITS: %s', tradeStyleCode, '${:,.2f}'.format(invoiceTotal))
        logger.info('**************************************************')
    else:
        logger.info('Total dollar amount for %s INVOICES: %s', tradeStyleCode, '${:,.2f}'.format(invoiceTotal))
        logger.info('**************************************************')
        
    if(creditMemo):
        aType = 'C'
    else:
        aType = 'I'
    writer.writerow(['TRL', CLIENT_CODE, tradeStyleCode, assignmentNumber, d, invoiceCount, aType, invoiceTotal])
    
    #Loop over each invoice header record..
    logger.info('<b>%s</b> \t <b>%s</b> \t <b>Amount</b>', 'Invoice'.ljust(10), 'Customer'.ljust(40))

    ###  GR PATCHES  ###
    if(creditMemo):
        q = session.query(InvoiceHeader).filter(InvoiceHeader.RDATLS == d, InvoiceHeader.RFCT.in_(VALID_TRADE_STYLES), InvoiceHeader.RWTF < 0, ~InvoiceHeader.RRNR.in_(backouts))
    else:
        q = session.query(InvoiceHeader).filter(InvoiceHeader.RDATLS == d, InvoiceHeader.RFCT.in_(VALID_TRADE_STYLES), InvoiceHeader.RWTF > 0, ~InvoiceHeader.RRNR.in_(backouts))
    ###  GR PATCHES  ###
        
    for row in q:
        logger.debug('Processing invoice %s', row.RRNR)
        line = ['DTL', CLIENT_CODE, tradeStyleCode, assignmentNumber, d]
        line.append(row.RRNR)  #Invoice Number; DB length = 10, WF will accept 22
        invoices.append(row.RRNR)
        line.append(row.RDATRE)        
        line.append(row.RWTF)                       #Invoice amount;         DB length = (15,5), WF will accept (12,2)
        line.append('C')                            #Terms type, constant field determined based on docs from WF
        line.append(row.RTXZ[2:])                   #Our Terms Code;         DB length =  5, WF will accept 15
        if(len(TERMS_CODES[row.RTXZ[2:]]) > 30):
            logger.warning('Terms description \'%s\' exceeds Wells Fargo character limit of 30! Sending \'%s\' instead.', TERMS_CODES[row.RTXZ[2:]], TERMS_CODES[row.RTXZ[2:]][:30])
        line.append(TERMS_CODES[row.RTXZ[2:]][:30]) #Terms Description;      DB length = 65, WF will accept 30
        line.append(row.RKDR)                       #Bill to account number; DB length = 10, WF will accept 20
        
        ''' Grab customer contact info '''
        logger.debug('SELECT * FROM TXUYKD00 WHERE TXUYKD00.KKDN = \'%s\'', row.RKDR)
        q2 = session.query(CustomerMaster).filter(CustomerMaster.KKDN == row.RKDR)
        if(q2.count() != 1):
            logger.fatal('Failed to retrieve customer information from customer master table. Customer Number: %s Rows Returned: %s', row.RKDR, q2.count())
            logger.fatal('**HALTING SCRIPT**')
            logger.fatal('**NOTHING HAS BEEN SENT TO WELLS FARGO FOR %s**', d)
            logger.debug('<--------------------------------------------------SCRIPT STOPPED---------------------------------------->')
            sys.exit('Failed to retrieve customer information from customer master table.')
        else:
            for row2 in q2:
                logger.info('%s \t %s \t %s', row.RRNR, row2.KNA2[:40].ljust(40), "{0:.2f}".format(row.RWTF).rjust(9))
                if(len(row2.KNA2.replace(',',' ')) > 40):
                    logger.warning('Bill to account name \'%s\' exceeds Wells Fargo character limit of 40! Sending \'%s\' instead.', row2.KNA2.replace(',',' '), row2.KNA2.replace(',',' ')[:40])
                line.append(row2.KNA2.replace(',',' ')[:40])  #Bill to account name; DB length = 65, WF will accept 40
                
                if(len(row2.KNA3.replace(',',' ')) > 40):
                    logger.warning('Bill to address line \'%s\' exceeds Wells Fargo character limit of 40! Sending \'%s\' instead.', row2.KNA3.replace(',',' '), row2.KNA3.replace(',',' ')[:40])     
                line.append(row2.KNA3.replace(',',' ')[:40])  #Bill to addr1;        DB length = 65, WF will accept 40
                
                if(len(row2.KNA4.replace(',',' ')) > 40):
                    logger.warning('Bill to address line \'%s\' exceeds Wells Fargo character limit of 40! Sending \'%s\' instead.', row2.KNA4.replace(',',' '), row2.KNA4.replace(',',' ')[:40])
                line.append(row2.KNA4.replace(',',' ')[:40])  #Bill to addr2;        DB length = 65, WF will accept 40
                
                line.append(row2.KORT.replace(',',' '))       #Bill to city;         DB length = 28, WF will accept 40
                line.append(row2.KSTA.replace(',',' '))       #Bill to state;        DB length =  3, WF will accept  3
                line.append(row2.KLND.replace(',',' '))       #Bill to country;      DB length =  3, WF will accept  3
                if(row2.KPLZ == '0'):
                    line.append('')
                else:
                    line.append(row2.KPLZ.replace(',',' '))   #Bill to zip,          DB length = 10, WF will accept 10
                    
                #Give it our best shot by stripping all non-digit characters first
                p = re.sub('[\D\s]', '', row2.KTEL)
                
                #Regex pattern tests if we have EXACTLY 10 DIGITS as required by Wells Fargo
                test = re.match(PHONE_NUM_PATTERN, p, flags=0)
                if(test == None):
                    line.append('') #Match failed, field is optional. Append empty string and move on
                else:                 
                    line.append(p)  #Bill to phone [optional]
        
        #if(int(row.RVAL) == 0):
            #line.append('')         #Wells Fargo requires 8 characters if anything is to be sent at all. So we'll catch the 0 from the DB here and just insert an empty string
        #else:
            #line.append(row.RVAL)   #As of date, dates in Jomar are already stored in the proper yyyymmdd format

        line.append('') #Replaced 
            
        line.append('')             #Intentionally left blank, "Store Number" column is N/A
        
        ''' Customer PO is optional as far as Wells-Fargo is concerned. We'll attempt to look it up '''
        logger.debug('SELECT * FROM TXUYUV00 WHERE TXUYUV00.UBLN = \'%s\'', row.RBLN)
        q2 = session.query(OrderHeader).filter(OrderHeader.UBLN == row.RBLN)
        if(q2.count() != 1):
            logger.error('Failed to retrieve customer PO number from order header table. Order Number on Invoice: %s Rows Returned: %s', row.RBLN, q2.count())
        else:
            for row2 in q2:
                line.append(row2.UIHR.replace(',',' ')) #Customer PO; DB length = 15, WF will accept 15
        
        writer.writerow(line)
    
    logger.info('')      
    f.close()
    return invoices

'''
For the purposes of running this via a web cgi interface I broke out the main logic of this script
into a 'main' function. See send_wf.py

iMode - a boolean, if True will prompt user for relevant info when running from a terminal
d - YYYYmmdd formatted date
sendNow - if True, the generated csv files will be sent immediatley to WF
email - if True the generated csv files are e-mailed to ar@example.com
'''
def main(iMode, d, sendNow, email=False):
    global session
    global backouts
    global MESSAGE
    os.chdir('D:\FTP')
    initializeLogger('wf_assignment_build')

    logger.debug('<--------------------------------------------------SCRIPT STARTED---------------------------------------->')
    logger.info('Running for date: %s', d)
    session = loadSession()

    #DL terms table to local dict for speedy lookup
    logger.debug('Downloading terms code list.')
    logger.debug('SELECT * FROM TXMYTX00 WHERE TXMYTX00.AXANR LIKE \'#Z%\'')
    q = session.query(TXMYTX00).filter(TXMYTX00.AXANR.like('#Z%'))
    for row in q:
        if(row.AXANR == '#Z'):
            continue
        else:
            TERMS_CODES[row.AXANR[2:]] = row.AXTXT[:65].strip().replace(',',' ')
    
    #Figure out which trade styles need assignments for given date
    tradeStyles = []

    ###  GR PATCHES  ###
    '''
    logger.debug('SELECT DISTINCT RFCT FROM TXUYRE00 WHERE TXUYRE00.RDATLS = \'%s\'', d)
    for value in session.query(InvoiceHeader.RFCT).filter(InvoiceHeader.RDATLS == d).distinct():
        tradeStyles.append(value.RFCT)
    '''
    tradeStyles.append('888888')
    ###  GR PATCHES  ###
        
    logger.info('Invoices/Credits where created on %s for these trade styles: %s', d, tradeStyles)

    #Get list of backed out invoices to ignore
    logger.debug('Fetching list of backed out invoices for %s', d)
    filter_string = 'b_create_date = ' + str(d)
    logger.debug('SELECT * FROM Invoice_Backouts WHERE %s;', filter_string)
    q = session.query(Invoice_Backouts).filter(Invoice_Backouts.b_create_date == d)
    for row in q:
        if row.o_create_date == d: #Check that create date of original invoice is also today
            logger.warn('Invoice %s was created and backed out with invoice %s on the same day. These will not be transmitted!', row.o_Invoice, row.b_Invoice)
            backouts.append(row.o_Invoice)
            backouts.append(row.b_Invoice)

    #Create assignment for each trade style
    processed_invoices = []
    for t in tradeStyles:
        processed_invoices += createAssignment(d, t, False)
        processed_invoices += createAssignment(d, t, True)
        
    if(iMode):
        if(raw_input('Send assignments to Wells Fargo? (n): ').lower() == 'y'):
            sendFiles(fileNames)
            if not TEST_MODE:
                logger.info('Marking %s Invoices/Credits as processed...', len(processed_invoices))
                ex = update(InvoiceHeader.__table__).where(InvoiceHeader.RRNR.in_(processed_invoices)).values(RFIL=u"X")
                session.execute(ex)
                session.commit()
                sts = subprocess.call("python D:\\AS400_Invoice_Interface.py " + d, shell=True)
    else:
        if(sendNow):
            sendFiles(fileNames)
            if not TEST_MODE:
                logger.info('Marking %s Invoices/Credits as processed...', len(processed_invoices))
                ex = update(InvoiceHeader.__table__).where(InvoiceHeader.RRNR.in_(processed_invoices)).values(RFIL=u"X")
                session.execute(ex)
                session.commit()
    if(email):
        try:
            logger.info('E-mailing generated CSVs to ar@example.com.')
            print '<b>THIS MAY TAKE A MOMENT **DO NOT** REFRESH YOUR BROWSER</b> :)'
            attachment = []
            for f in fileNames:
                attachment.append('D:\\FTP\\SEND_WF\\' + f)

            send_alert_html('ar@example.com', 'Wells Fargo Assignments', message=MESSAGE.getvalue(), attachments=attachment)
        except:
            logger.warn('Unable to e-mail CSV files. Is the internet down?')
            logger.warn('The CSVs can be found in D:\\FTP\\SEND_WF on the test server')
            logger.debug(sys.exc_info()[0])
            
    print 'See the <a href="http://stmtest:81/cgi-bin/logs.py?logname=wf">log file</a> for extra details...'
    session.close()
    logger.debug('<--------------------------------------------------SCRIPT STOPPED---------------------------------------->')
    MESSAGE.truncate(0)

    if(sendNow):
        sts = subprocess.call("python D:\\AS400_Invoice_Interface.py " + d, shell=True)

if __name__ == '__main__':
    #main(False, 20161108, False, email=True)
    #sys.exit()
    
    a = len(sys.argv)
    interactive_mode = False
    if(a == 2):
        if(len(sys.argv[1]) == 8): #grab date from sys.argv[1] and run
            DATE = int(sys.argv[1])
        else:
            sys.exit('Invalid date supplied ' + sys.argv[1] + '. Must be in yyyymmdd format.')
    else:
        interactive_mode = True
        prompt = time.strftime("Date (%Y%m%d): ")
        DATE = raw_input(prompt)
        if(len(DATE) == 0):
            DATE = int(time.strftime("%Y%m%d"))
        elif(len(DATE) == 8):
            d = time.strptime(DATE, "%Y%m%d")
            prompt = datetime.fromtimestamp(time.mktime(d)).strftime("Generate assignments for %A, %B %d %Y? (n): ")
            confirm = raw_input(prompt)
            if(confirm == 'y' or confirm == 'Y'):
                DATE = int(DATE)
            else:
                sys.exit('User cancelled script.')
        else:
            sys.exit('Date format doesn\'t look right... Exiting.')

    main(interactive_mode, DATE, True)