wes/Python_Examples
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Found more code examples from STM. Used claude to generate a summary (CLAUDE.md)

Browse Source
This commit is contained in:
wes
2026-02-11 06:05:20 -05:00
parent cf0edba776
commit c4b970cbde
33 changed files with 6077 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
stm_etl/FTP/wf_assignments.py
+483
View File
@@ -0,0 +1,483 @@
'''
Created on Jan 7, 2016
Generate assignment of invoices to be sent to Wells Fargo in csv format.
This is mainly called from the send_wf.py cgi script running on the Apache instance on our test server.
'WF' in comments = Wells Fargo
@todo: Properly truncate invoice amount value to fit WF's decimal(12,2) field size. TOTAL and DETAIL records!
@author: Wes
'''
import paramiko, csv, sys, logging, re, time, os, smtplib, sqlite3, StringIO, subprocess
from datetime import datetime
from sqlalchemy import create_engine, func, Table, text, Column
from sqlalchemy.types import VARCHAR
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy.sql.expression import update
from alert import send_alert_html
TEST_MODE = False #When 'True' files will be uploaded to the 'testin' directory on the remote server
''' CLIENT_CODE = 5867 #Old Sunbury Textile Mills client code. Last Used: 09/29/2017 '''
CLIENT_CODE = 4143 #New code assigned to us by WF. Effective: 10/02/2017
REMOTE_HOST = 'sftp.spscommerce.net'
REMOTE_PORT = 10022
MESSAGE = None
VALID_TRADE_STYLES = ['888888', '88888800', '88888803', '88888805', '88888809']
'''
The only valid trade style numbers at the time this pattern was created are:
888888
88888800
88888803
88888805
88888809
'''
TRADE_STYLE_PATTERN = "888888(?P<tradeStyle>\d{0}$|0[0359]$)"
''' EXACTLY 10 digits '''
PHONE_NUM_PATTERN = "^\d{10}$"
#Moved connection string into sqlite db
for row in sqlite3.connect('D:\\FTP\\ftp.db').execute("SELECT * FROM connection_strings WHERE code = 'wfa'"):
engine = create_engine(row[1], echo=False)
Base = declarative_base(engine)
metadata = Base.metadata
logger = None
LOG_LEVEL = logging.DEBUG
TERMS_CODES = { '00': 'No Terms' }
assignmentNumber = 0
fileNames = []
backouts = ['9999999999']
#Customer Master
class CustomerMaster(Base):
__tablename__ = 'TXUYKD00'
__table_args__ = {'autoload':True}
#Tables and Codes
class TXMYTX00(Base):
__tablename__ = 'TXMYTX00'
__table_args__ = {'autoload':True}
#Invoice header table
class InvoiceHeader(Base):
__tablename__ = 'TXUYRE00'
__table_args__ = {'autoload':True}
#Invoice detail table
class InvoiceDetail(Base):
__tablename__ = 'TXUYRE01'
__table_args__ = {'autoload':True}
#Order Header
class OrderHeader(Base):
__tablename__ = 'TXUYUV00'
__table_args__ = {'autoload':True}
#Backouts
class Invoice_Backouts(Base):
__table__ = Table('Invoice_Backouts', Base.metadata,
Column('b_Invoice', VARCHAR, primary_key=True),
autoload=True, autoload_with=engine
)
'''
Storing FTP site information in a sqlite db now to keep it out of source code
'''
def load_ftp_credentials():
vault = sqlite3.connect('D:\\FTP\\ftp.db')
cursor = vault.execute("SELECT * from ftp_auth where code='wf'")
for row in cursor:
return row
'''
Upload files to WF's ftp site for processing.
Global var TEST_MODE dictates which remote dir the files are uploaded into.
files should contain a list of fully qualified file paths to be sent.
'''
def sendFiles(files):
return
# global logger
# global TEST_MODE
# if(TEST_MODE):
# remoteDir = '/testin/'
# else:
# remoteDir = '/in/'
#
# creds = load_ftp_credentials()
#
# logger.info('Opening SFTP connection to %s:%s',REMOTE_HOST,REMOTE_PORT)
# transport = paramiko.Transport((REMOTE_HOST, REMOTE_PORT))
# transport.connect(username = creds[3], password = creds[4])
# sftp = paramiko.SFTPClient.from_transport(transport)
#
# for f in files:
# path = remoteDir + f
# logger.info('sftp.put(\'%s\', \'%s\')', ('D:\\FTP\\SEND_WF\\' + f), path)
# sftp.put(('D:\\FTP\\SEND_WF\\' + f), path)
#
# logger.debug('DIR %s = %s', remoteDir, sftp.listdir(remoteDir[:-1]))
#
# logger.info('Closing SFTP connection.')
# sftp.close()
# transport.close()
'''
Opens connection to database returning a session object.
'''
def loadSession():
global engine
Session = sessionmaker(bind=engine)
session = Session()
return session
def initializeLogger(name='log'):
global logger
global MESSAGE
logger = logging.getLogger(__name__)
logger.setLevel(LOG_LEVEL)
handler = logging.FileHandler('logs\\' + name + '.log')
handler.setLevel(LOG_LEVEL)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
#Also send logs > DEBUG to stdout
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
#Capture log messages in a string for e-mail alert
MESSAGE = StringIO.StringIO()
sh = logging.StreamHandler(MESSAGE)
sh.setLevel(logging.INFO)
sh.setFormatter(formatter)
logger.addHandler(sh)
'''
The result of this function is a single assignment file.
An assignment containing all of the invoices (or credits) for a single tradestyle on a given date.
Invoices and Credits (assuming all is done through the ERP method in jomar) are stored in the same
table.
An 'Invoice' for the purposes of transmitting to wf is defined as any invoice in the table with a
_positive_ dollar amount.
A 'Credit' is similarly defined as any invoice with a _negative_ dolar amount.
There is a flag cloumn in the table that designates what type of invoice it is (SO, SR, etc.) however
Caleb has expressed that the +/- methodology works well.
Valid Tradestyles: 00, 03, 05, 09
These are mapped to these factor accounts in Jomar: 888888, 88888803, 88888805, 88888809
d - the target date in YYYmmdd format
tStyle - The associated factor account for the target tradestyle ex. 888888 for 00
creditMemo - True to process credits, False to process invoices
ex: createAssignment(20160520, '888888', False) # Generate 00 invoice assignment for 05/20/16
This method returns a list containing the invoice/credit numbers that were processed. This is
used by the main method to build a complete list of invoices to flag as processed if the CSV files
are sent to Wells Fargo. This prevents any invoices processed by this script from being "backed out"
after the fact.
'''
def createAssignment(d, tStyle, creditMemo):
global assignmentNumber
global fileNames
global backouts
invoices = []
#Check for invalid trade styles, The script ignores invalid trade styles and proceeds if it can.
test = re.match(TRADE_STYLE_PATTERN, tStyle, flags=0)
if(test == None):
if not(tStyle == ''):
logger.warning('Ignoring Invalid Trade Style: \'%s\'', tStyle)
return invoices
elif(test.group('tradeStyle') == ''):
tradeStyleCode = '01'
logger.debug('Jomar trade style = %s --> Wells Fargo trade style = %s', tStyle, tradeStyleCode)
else:
tradeStyleCode = test.group('tradeStyle')
logger.debug('Jomar trade style = %s --> Wells Fargo trade style = %s', tStyle, tradeStyleCode)
#Get number of invoices for this trade style
#logger.debug('SELECT COUNT(RRNR) FROM TXUYRE00 WHERE RDATLS = \'%s\' AND RFCT LIKE \'%s\'', d, tStyle)
### GR PATCHES ###
if(creditMemo):
logger.debug('SELECT COUNT(RRNR) FROM TXUYRE00 WHERE RDATLS = \'%s\' AND RFCT LIKE \'%s\' AND RWTF < 0;', d, tStyle)
invoiceCount = session.query(InvoiceHeader.RRNR).filter(InvoiceHeader.RDATLS == d, InvoiceHeader.RFCT.in_(VALID_TRADE_STYLES), InvoiceHeader.RWTF < 0, ~InvoiceHeader.RRNR.in_(backouts)).count()
else:
logger.debug('SELECT COUNT(RRNR) FROM TXUYRE00 WHERE RDATLS = \'%s\' AND RFCT LIKE \'%s\' AND RWTF > 0;', d, tStyle)
invoiceCount = session.query(InvoiceHeader.RRNR).filter(InvoiceHeader.RDATLS == d, InvoiceHeader.RFCT.in_(VALID_TRADE_STYLES), InvoiceHeader.RWTF > 0, ~InvoiceHeader.RRNR.in_(backouts)).count()
### GR PATCHES ###
if(invoiceCount == 0):
return invoices
if(creditMemo):
logger.info('**************************************************')
logger.info('Total number of %s CREDITS: %s', tradeStyleCode, invoiceCount)
else:
logger.info('**************************************************')
logger.info('Total number of %s INVOICES: %s', tradeStyleCode, invoiceCount)
#Trade style is valid, increment assignment
assignmentNumber += 1
#Open file for writing
fname = str(CLIENT_CODE) + tradeStyleCode + '_INV_' + str(d) + '_' + str(assignmentNumber) + '.csv'
logger.debug('Creating %s. Using \'excel\' dialect for csv writer.', fname)
f = open('SEND_WF\\' + fname, 'wb')
writer = csv.writer(f, dialect='excel')
fileNames.append(fname)
#Get total dollar amount of all invoices for this trade style
invoiceTotal = 0
#logger.debug('SELECT SUM(RWTS) sum FROM TXUYRE00 WHERE RDATLS = \'%s\' AND RFCT LIKE \'%s\'', d, tStyle)
### GR PATCHES ###
if(creditMemo):
logger.debug('SELECT SUM(RWTS) sum FROM TXUYRE00 WHERE RDATLS = \'%s\' AND RFCT LIKE \'%s\' AND RWTF < 0;', d, tStyle)
for row in session.query(func.sum(InvoiceHeader.RWTS).label("sum")).filter(InvoiceHeader.RDATLS == d, InvoiceHeader.RFCT.in_(VALID_TRADE_STYLES), InvoiceHeader.RWTF < 0, ~InvoiceHeader.RRNR.in_(backouts)):
invoiceTotal = row.sum
else:
logger.debug('SELECT SUM(RWTS) sum FROM TXUYRE00 WHERE RDATLS = \'%s\' AND RFCT LIKE \'%s\' AND RWTF > 0;', d, tStyle)
for row in session.query(func.sum(InvoiceHeader.RWTS).label("sum")).filter(InvoiceHeader.RDATLS == d, InvoiceHeader.RFCT.in_(VALID_TRADE_STYLES), InvoiceHeader.RWTF > 0, ~InvoiceHeader.RRNR.in_(backouts)):
invoiceTotal = row.sum
### GR PATCHES ###
if(creditMemo):
logger.info('Total dollar amount for %s CREDITS: %s', tradeStyleCode, '${:,.2f}'.format(invoiceTotal))
logger.info('**************************************************')
else:
logger.info('Total dollar amount for %s INVOICES: %s', tradeStyleCode, '${:,.2f}'.format(invoiceTotal))
logger.info('**************************************************')
if(creditMemo):
aType = 'C'
else:
aType = 'I'
writer.writerow(['TRL', CLIENT_CODE, tradeStyleCode, assignmentNumber, d, invoiceCount, aType, invoiceTotal])
#Loop over each invoice header record..
logger.info('<b>%s</b> \t <b>%s</b> \t <b>Amount</b>', 'Invoice'.ljust(10), 'Customer'.ljust(40))
### GR PATCHES ###
if(creditMemo):
q = session.query(InvoiceHeader).filter(InvoiceHeader.RDATLS == d, InvoiceHeader.RFCT.in_(VALID_TRADE_STYLES), InvoiceHeader.RWTF < 0, ~InvoiceHeader.RRNR.in_(backouts))
else:
q = session.query(InvoiceHeader).filter(InvoiceHeader.RDATLS == d, InvoiceHeader.RFCT.in_(VALID_TRADE_STYLES), InvoiceHeader.RWTF > 0, ~InvoiceHeader.RRNR.in_(backouts))
### GR PATCHES ###
for row in q:
logger.debug('Processing invoice %s', row.RRNR)
line = ['DTL', CLIENT_CODE, tradeStyleCode, assignmentNumber, d]
line.append(row.RRNR) #Invoice Number; DB length = 10, WF will accept 22
invoices.append(row.RRNR)
line.append(row.RDATRE)
line.append(row.RWTF) #Invoice amount; DB length = (15,5), WF will accept (12,2)
line.append('C') #Terms type, constant field determined based on docs from WF
line.append(row.RTXZ[2:]) #Our Terms Code; DB length = 5, WF will accept 15
if(len(TERMS_CODES[row.RTXZ[2:]]) > 30):
logger.warning('Terms description \'%s\' exceeds Wells Fargo character limit of 30! Sending \'%s\' instead.', TERMS_CODES[row.RTXZ[2:]], TERMS_CODES[row.RTXZ[2:]][:30])
line.append(TERMS_CODES[row.RTXZ[2:]][:30]) #Terms Description; DB length = 65, WF will accept 30
line.append(row.RKDR) #Bill to account number; DB length = 10, WF will accept 20
''' Grab customer contact info '''
logger.debug('SELECT * FROM TXUYKD00 WHERE TXUYKD00.KKDN = \'%s\'', row.RKDR)
q2 = session.query(CustomerMaster).filter(CustomerMaster.KKDN == row.RKDR)
if(q2.count() != 1):
logger.fatal('Failed to retrieve customer information from customer master table. Customer Number: %s Rows Returned: %s', row.RKDR, q2.count())
logger.fatal('**HALTING SCRIPT**')
logger.fatal('**NOTHING HAS BEEN SENT TO WELLS FARGO FOR %s**', d)
logger.debug('<--------------------------------------------------SCRIPT STOPPED---------------------------------------->')
sys.exit('Failed to retrieve customer information from customer master table.')
else:
for row2 in q2:
logger.info('%s \t %s \t %s', row.RRNR, row2.KNA2[:40].ljust(40), "{0:.2f}".format(row.RWTF).rjust(9))
if(len(row2.KNA2.replace(',',' ')) > 40):
logger.warning('Bill to account name \'%s\' exceeds Wells Fargo character limit of 40! Sending \'%s\' instead.', row2.KNA2.replace(',',' '), row2.KNA2.replace(',',' ')[:40])
line.append(row2.KNA2.replace(',',' ')[:40]) #Bill to account name; DB length = 65, WF will accept 40
if(len(row2.KNA3.replace(',',' ')) > 40):
logger.warning('Bill to address line \'%s\' exceeds Wells Fargo character limit of 40! Sending \'%s\' instead.', row2.KNA3.replace(',',' '), row2.KNA3.replace(',',' ')[:40])
line.append(row2.KNA3.replace(',',' ')[:40]) #Bill to addr1; DB length = 65, WF will accept 40
if(len(row2.KNA4.replace(',',' ')) > 40):
logger.warning('Bill to address line \'%s\' exceeds Wells Fargo character limit of 40! Sending \'%s\' instead.', row2.KNA4.replace(',',' '), row2.KNA4.replace(',',' ')[:40])
line.append(row2.KNA4.replace(',',' ')[:40]) #Bill to addr2; DB length = 65, WF will accept 40
line.append(row2.KORT.replace(',',' ')) #Bill to city; DB length = 28, WF will accept 40
line.append(row2.KSTA.replace(',',' ')) #Bill to state; DB length = 3, WF will accept 3
line.append(row2.KLND.replace(',',' ')) #Bill to country; DB length = 3, WF will accept 3
if(row2.KPLZ == '0'):
line.append('')
else:
line.append(row2.KPLZ.replace(',',' ')) #Bill to zip, DB length = 10, WF will accept 10
#Give it our best shot by stripping all non-digit characters first
p = re.sub('[\D\s]', '', row2.KTEL)
#Regex pattern tests if we have EXACTLY 10 DIGITS as required by Wells Fargo
test = re.match(PHONE_NUM_PATTERN, p, flags=0)
if(test == None):
line.append('') #Match failed, field is optional. Append empty string and move on
else:
line.append(p) #Bill to phone [optional]
#if(int(row.RVAL) == 0):
#line.append('') #Wells Fargo requires 8 characters if anything is to be sent at all. So we'll catch the 0 from the DB here and just insert an empty string
#else:
#line.append(row.RVAL) #As of date, dates in Jomar are already stored in the proper yyyymmdd format
line.append('') #Replaced
line.append('') #Intentionally left blank, "Store Number" column is N/A
''' Customer PO is optional as far as Wells-Fargo is concerned. We'll attempt to look it up '''
logger.debug('SELECT * FROM TXUYUV00 WHERE TXUYUV00.UBLN = \'%s\'', row.RBLN)
q2 = session.query(OrderHeader).filter(OrderHeader.UBLN == row.RBLN)
if(q2.count() != 1):
logger.error('Failed to retrieve customer PO number from order header table. Order Number on Invoice: %s Rows Returned: %s', row.RBLN, q2.count())
else:
for row2 in q2:
line.append(row2.UIHR.replace(',',' ')) #Customer PO; DB length = 15, WF will accept 15
writer.writerow(line)
logger.info('')
f.close()
return invoices
'''
For the purposes of running this via a web cgi interface I broke out the main logic of this script
into a 'main' function. See send_wf.py
iMode - a boolean, if True will prompt user for relevant info when running from a terminal
d - YYYYmmdd formatted date
sendNow - if True, the generated csv files will be sent immediatley to WF
email - if True the generated csv files are e-mailed to ar@example.com
'''
def main(iMode, d, sendNow, email=False):
global session
global backouts
global MESSAGE
os.chdir('D:\FTP')
initializeLogger('wf_assignment_build')
logger.debug('<--------------------------------------------------SCRIPT STARTED---------------------------------------->')
logger.info('Running for date: %s', d)
session = loadSession()
#DL terms table to local dict for speedy lookup
logger.debug('Downloading terms code list.')
logger.debug('SELECT * FROM TXMYTX00 WHERE TXMYTX00.AXANR LIKE \'#Z%\'')
q = session.query(TXMYTX00).filter(TXMYTX00.AXANR.like('#Z%'))
for row in q:
if(row.AXANR == '#Z'):
continue
else:
TERMS_CODES[row.AXANR[2:]] = row.AXTXT[:65].strip().replace(',',' ')
#Figure out which trade styles need assignments for given date
tradeStyles = []
### GR PATCHES ###
'''
logger.debug('SELECT DISTINCT RFCT FROM TXUYRE00 WHERE TXUYRE00.RDATLS = \'%s\'', d)
for value in session.query(InvoiceHeader.RFCT).filter(InvoiceHeader.RDATLS == d).distinct():
tradeStyles.append(value.RFCT)
'''
tradeStyles.append('888888')
### GR PATCHES ###
logger.info('Invoices/Credits where created on %s for these trade styles: %s', d, tradeStyles)
#Get list of backed out invoices to ignore
logger.debug('Fetching list of backed out invoices for %s', d)
filter_string = 'b_create_date = ' + str(d)
logger.debug('SELECT * FROM Invoice_Backouts WHERE %s;', filter_string)
q = session.query(Invoice_Backouts).filter(Invoice_Backouts.b_create_date == d)
for row in q:
if row.o_create_date == d: #Check that create date of original invoice is also today
logger.warn('Invoice %s was created and backed out with invoice %s on the same day. These will not be transmitted!', row.o_Invoice, row.b_Invoice)
backouts.append(row.o_Invoice)
backouts.append(row.b_Invoice)
#Create assignment for each trade style
processed_invoices = []
for t in tradeStyles:
processed_invoices += createAssignment(d, t, False)
processed_invoices += createAssignment(d, t, True)
if(iMode):
if(raw_input('Send assignments to Wells Fargo? (n): ').lower() == 'y'):
sendFiles(fileNames)
if not TEST_MODE:
logger.info('Marking %s Invoices/Credits as processed...', len(processed_invoices))
ex = update(InvoiceHeader.__table__).where(InvoiceHeader.RRNR.in_(processed_invoices)).values(RFIL=u"X")
session.execute(ex)
session.commit()
sts = subprocess.call("python D:\\AS400_Invoice_Interface.py " + d, shell=True)
else:
if(sendNow):
sendFiles(fileNames)
if not TEST_MODE:
logger.info('Marking %s Invoices/Credits as processed...', len(processed_invoices))
ex = update(InvoiceHeader.__table__).where(InvoiceHeader.RRNR.in_(processed_invoices)).values(RFIL=u"X")
session.execute(ex)
session.commit()
if(email):
try:
logger.info('E-mailing generated CSVs to ar@example.com.')
print '<b>THIS MAY TAKE A MOMENT **DO NOT** REFRESH YOUR BROWSER</b> :)'
attachment = []
for f in fileNames:
attachment.append('D:\\FTP\\SEND_WF\\' + f)
send_alert_html('ar@example.com', 'Wells Fargo Assignments', message=MESSAGE.getvalue(), attachments=attachment)
except:
logger.warn('Unable to e-mail CSV files. Is the internet down?')
logger.warn('The CSVs can be found in D:\\FTP\\SEND_WF on the test server')
logger.debug(sys.exc_info()[0])
print 'See the <a href="http://stmtest:81/cgi-bin/logs.py?logname=wf">log file</a> for extra details...'
session.close()
logger.debug('<--------------------------------------------------SCRIPT STOPPED---------------------------------------->')
MESSAGE.truncate(0)
if(sendNow):
sts = subprocess.call("python D:\\AS400_Invoice_Interface.py " + d, shell=True)
if __name__ == '__main__':
#main(False, 20161108, False, email=True)
#sys.exit()
a = len(sys.argv)
interactive_mode = False
if(a == 2):
if(len(sys.argv[1]) == 8): #grab date from sys.argv[1] and run
DATE = int(sys.argv[1])
else:
sys.exit('Invalid date supplied ' + sys.argv[1] + '. Must be in yyyymmdd format.')
else:
interactive_mode = True
prompt = time.strftime("Date (%Y%m%d): ")
DATE = raw_input(prompt)
if(len(DATE) == 0):
DATE = int(time.strftime("%Y%m%d"))
elif(len(DATE) == 8):
d = time.strptime(DATE, "%Y%m%d")
prompt = datetime.fromtimestamp(time.mktime(d)).strftime("Generate assignments for %A, %B %d %Y? (n): ")
confirm = raw_input(prompt)
if(confirm == 'y' or confirm == 'Y'):
DATE = int(DATE)
else:
sys.exit('User cancelled script.')
else:
sys.exit('Date format doesn\'t look right... Exiting.')
main(interactive_mode, DATE, True)