''' Created on Sep 10, 2013 @author: Wesley Ray Replacement for the XMLmapper software purchased a few years ago. The current one has too many driver issues/conflicts and I knew that parsing XML with Python is pretty easy. This makes use of xml.etree.ElementTree to create a Document Object Model (DOM) of the input files. This makes it very easy to locate information if you know the the tag structure; which we do from the .ini file of the old XMLmapper program. Lucky that programmer decided against hard-coding that information into the binaries. Taking the DOM approach is inherently slow (the entire file is loaded into memory and parsed). It's not a major problem for our purposes as the XML files are < 10MB, it still does take a moment to load the XML file initially. The parsing and uploading of the data afterwards is almost instantaneous however. ''' from STM.TabelModel import FileTableModel from com.ibm.as400.access import AS400JDBCDriver from java.awt import BorderLayout, Cursor from java.io import ByteArrayInputStream, File from java.sql import DriverManager from javax.imageio import ImageIO from javax.swing import JFrame, UIManager, JTable, JScrollPane, JButton, JPanel, \ JOptionPane, ImageIcon import base64 import datetime from datetime import timedelta import glob import java.lang.Exception as JavaException import os, getpass import xml.etree.ElementTree as ET VERSION_ID = '1.6' NETWORK_PATH = '\\\\EATSRV\\XML Files' LOG_FOLDER_DEPTH = NETWORK_PATH.count('\\') + 2; LOG_FILE_REGEX = '[0-9][0-9]-[0-9][0-9]-[0-9][0-9][0-9][0-9].txt' KEEP_LOG_FILES = 7 #Number of days to keep log files in the log folder DEBUG_MODE = True #Set this True to write to test database and debug log file ASCII_ART = [' .d8888b. 88888888888 888b d888', ' d88P Y88b 888 8888b d8888', ' Y88b. 888 88888b.d88888', ' "Y888b. 888 888Y88888P888', ' "Y88b. 888 888 Y888P 888', ' "888 888 888 Y8P 888', ' Y88b d88P 888 888 " 888', ' "Y8888P" 888 888 888'] ''' Invoke QCMDEXC via a stored procedure (Thanks IBM!) so that we can get the EAT data into the system right away. This runs a CL program I've written on the AS400 called EATPARSE. The source code for EATPARSE is located in STMLIB/QCLLESRC ''' FORCE_PROCESSING = "call qsys.qcmdexc('CALL STMLIB/EATPARSE',0000000020.00000)" #EATFILEA PATTERN_NUMBER_TAG = 'Entries/TransferObject/PseudoColorImageTO/Name' # Maps to EATDATA/EATFILEA.EATNAM PICKS_TAG = 'Entries/TransferObject/PseudoColorImageTO/WeftDensity' # Maps to EATDATA/EATFILEA.EATPKS TIME_STAMP_TAG_A = 'Entries/TransferObject/PseudoColorImageTO/LastSavedTime' # Maps to EATDATA/EATFILEA.EATTMS HORIZ_WARP_ENDS_TAG = 'Entries/TransferObject/PseudoColorImageTO/SizeX' # Maps to EATDATA/EATFILEA.EATHWE VERT_REPEAT_TAG = 'Entries/TransferObject/PseudoColorImageTO/WorldSizeY' # Maps to EATDATA/EATFILEA.EATVRP NOTE_TAG = 'Entries/TransferObject/PseudoColorImageTO/Note' # Maps to EATDATA/EATFILEA.EATNOT #EATFILEB TOTAL_CARDS_TAG = 'Entries/Application/AddBoxMotion/BoxMotionInfo/BoxMotionTotalPicks' # Maps to EATDATA/EATFILEB.EATCDS #EATFILEC #Children of BoxMotionColorList are ListEntry tags SEQUENCE_NUM_TAG_C = 'Entries/Application/AddBoxMotion/BoxMotionInfo/BoxMotionList/ListEntry/BoxMotionListEntry/BoxMotionColorList' # Maps to EATDATA/EATFILEC.EATSEQ #ListEntry Tags have these as children CARD_COUNT_TAG = 'BoxMotionColorListEntry/ColorCount' # Maps to EATDATA/EATFILEC.EATCCT #EATFILED SEQUENCE_NUM_TAG_D = 'Entries/TransferObject/PseudoColorImageTO/ModificationHistoryList' # Maps to EATDATA/EATFILED.EATSQ5 ARTIST_TAG = 'ModificationHistoryListEntry/Artist' # Maps to EATDATA/EATFILED.EATART COMPUTER_NAME_TAG = 'ModificationHistoryListEntry/ComputerName' # Maps to EATDATA/EATFILED.EATCNM TIME_STAMP_TAG_D = 'ModificationHistoryListEntry/Time' # Maps to EATDATA/EATFILED.EATTMS #EATFILEE CAST_OUT_NAME_TAG = 'Entries/Application/MachineDefinition/MachDefCastOutName' # Maps to EATDATA/EATFILEE.EATDON #EATFILEF SEQUENCE_NUM_TAG_F = 'Entries/Application/CalcJCSimu/SimuYarnList/WeftYarnList' # Maps to EATDATA/EATFILEF.EATSEQ PAT_NUM_TAG_F = 'YarnName' # Maps to EATDATA/EATFILEF.EATNAM #EATFILEG YARN_NUM_TAG = 'Entries/Application/AddBoxMotion/BoxMotionInfo/BoxMotionYarnList/ListEntry/BoxMotionYarnListEntry/BoxMotionYarnNumber' # Maps to EATDATA/EATFILEG.EATYRN TISSUE_CARD_TAG = 'Entries/Application/AddBoxMotion/BoxMotionInfo/BoxMotionYarnList/ListEntry/BoxMotionYarnListEntry/BoxMotionYarnCountRegSingle' # Maps to EATDATA/EATFILEG.EATTCD GROUND_CARD_TAG = 'Entries/Application/AddBoxMotion/BoxMotionInfo/BoxMotionYarnList/ListEntry/BoxMotionYarnListEntry/BoxMotionYarnCountSingle' # Maps to EATDATA/EATFILEG.EATGCD if(DEBUG_MODE): #SQL statements FILE_A_INSERT = 'insert into WES.EATFILEA values(?, ?, ?, ?, ?, ?, ?, ?)' FILE_B_INSERT = 'insert into WES.EATFILEB values(?, ?, ?)' FILE_C_INSERT = 'insert into WES.EATFILEC values(?, ?, ?, ?)' FILE_D_INSERT = 'insert into WES.EATFILED values(?, ?, ?, ?, ?, ?)' FILE_E_INSERT = 'insert into WES.EATFILEE values(?, ?, ?)' FILE_F_INSERT = 'insert into WES.EATFILEF values(?, ?, ?, ?)' FILE_G_INSERT = 'insert into WES.EATFILEG values(?, ?, ?, ?, ?)' else: #SQL statements FILE_A_INSERT = 'insert into EATDATA.EATFILEA values(?, ?, ?, ?, ?, ?, ?, ?)' FILE_B_INSERT = 'insert into EATDATA.EATFILEB values(?, ?, ?)' FILE_C_INSERT = 'insert into EATDATA.EATFILEC values(?, ?, ?, ?)' FILE_D_INSERT = 'insert into EATDATA.EATFILED values(?, ?, ?, ?, ?, ?)' FILE_E_INSERT = 'insert into EATDATA.EATFILEE values(?, ?, ?)' FILE_F_INSERT = 'insert into EATDATA.EATFILEF values(?, ?, ?, ?)' FILE_G_INSERT = 'insert into EATDATA.EATFILEG values(?, ?, ?, ?, ?)' STATS_INSERT = 'insert into WES.XMLSTATS values(?, ?)' #Global Vars transferObjectSid = 0 applicationBoxMotionSid = 0 applicationMachineSid = 0 applicationCalcJCSimuSid = 0 patternNumber = '' connection = None fileAInsert = None fileBInsert = None fileCInsert = None fileDInsert = None fileEInsert = None fileFInsert = None fileGInsert = None statsInsert = None forceUpdate = None setStats = None frame = None qtm = None logFile = None currentlyProcessing = '' ''' Returns a BufferedImage object of the preview image of the pattern encoded in the XML Currently not used for anything, I wanted to make a nifty progress dialog that displayed a thumbnail of the pattern whose XML is currently being processed ''' def getPreviewImage(root): return ImageIO.read(ByteArrayInputStream(base64.decodestring(root.find('Entries/TransferObject/PseudoColorImageTO/Preview').text))) ''' This method is one of the reasons why I love Python. In one line we are opening (or creating if the file doesn't exist) a text file with the current date as the name. Example: 10-27-13.txt ''' def openLogFile(): if(DEBUG_MODE): return open(datetime.datetime.now().strftime(NETWORK_PATH + '\\logs\\%m-%d-%Y_DEBUG.txt'), 'a') # The 'a' signifies append mode, file can only be written to. Does not affect current contents of file. else: return open(datetime.datetime.now().strftime(NETWORK_PATH + '\\logs\\%m-%d-%Y.txt'), 'a') ''' This method simply prepends a time stamp to a string you wish to print to stdout. It also writes the resulting string out to the log file ''' def debug(msg): now = datetime.datetime.now() if(now.minute < 10): mins = '0' + str(now.minute) else: mins = str(now.minute) msg = '[' + str(now.hour) + ':' + mins + '] ' + msg print msg logFile.write(msg + '\n') ''' Modified version of debug that takes a Python Error as a parameter so that it can be printed to the stdout and written to the log file ''' def debugError(err): now = datetime.datetime.now() if(now.minute < 10): mins = '0' + str(now.minute) else: mins = str(now.minute) msg = '[' + str(now.hour) + ':' + mins + '] Error Message:' print msg, err logFile.write(msg + repr(err) + '\n') ''' Loads the IBM AS400 JDBC driver and opens a connection with the AS400 ''' def connect(): DriverManager.registerDriver(AS400JDBCDriver()) return DriverManager.getConnection('jdbc:as400://SUNBURY', 'USER', 'PASSWORD') ''' Opens a connection to the AS400, global var 'connection' gets the reference to the connection object pre-compiles SQL statements and assigns their references to the appropriate global variables ''' def prepareToSend(): global connection global fileAInsert global fileBInsert global fileCInsert global fileDInsert global fileEInsert global fileFInsert global fileGInsert global forceUpdate global statsInsert debug('Connecting to AS400 as user: EAT') connection = connect() #Open a connection debug('Pre-compiling SQL statements') fileAInsert = connection.prepareStatement(FILE_A_INSERT) fileBInsert = connection.prepareStatement(FILE_B_INSERT) fileCInsert = connection.prepareStatement(FILE_C_INSERT) fileDInsert = connection.prepareStatement(FILE_D_INSERT) fileEInsert = connection.prepareStatement(FILE_E_INSERT) fileFInsert = connection.prepareStatement(FILE_F_INSERT) fileGInsert = connection.prepareStatement(FILE_G_INSERT) statsInsert = connection.prepareStatement(STATS_INSERT) forceUpdate = connection.prepareCall(FORCE_PROCESSING) ''' This method does the actual load, parse, upload routine. Nothing profound here see the in-line comments in the method. [MODIFIED] 9/13/13 - It was bugging me the whole time that if a SQLException was thrown all operations for subsequent files would not occur; resulting in data being in some of the files (maybe a,b,c) and not in the rest which would likely be an issue for the job that picks up the data on the AS400. So I found out that java.sql.PreparedStatement has addBatch() and executeBatch() methods. So now all of the actual inserting can be done after the prepared statements are created. It is the setting of the parameters of the prepared statements that is most likely to throw an exception. If that occurs now, none of the files will be updated, instead of just the files leading up to the exception being updated. [MODIFIED] 9/17/13 - Wow.. I have seen the light. PreparedStatement.setObject() will do type conversion for me... No more type conversions, and setString, setInt, setYourFace... On a more serious note, this did work well and it really cleans up the code. This was easy to do partially because all of the data parsed from the XML files are strings. Strings can certainly be converted to integers and floats as long as they actually contain numbers. You would need to be careful in a language like Python to know what type of data is contained in a variable when passing it to setObject or you'll get SQLExceptions. ''' def parseAndSend(inFile): debug('Loading ' + inFile + '...') tree = ET.parse(inFile) debug('--------------------------------') root = tree.getroot() #Get root XML tag getSids(root) #Parse sid's into global vars debug('Parsing XML data for EATFILEA') aData = getFileAData(root) #Get data to insert into EATFILEA debug('Parsing XML data for EATFILEB') bData = root.find(TOTAL_CARDS_TAG).text #Data for EATFILEB is just one field debug('\t\tTotal Cards: ' + bData) debug('Parsing XML data for EATFILEC') cData = getFileCData(root) #Get data to insert into EATFILEC debug('Parsing XML data for EATFILED') dData = getFileDData(root) #Get data to insert into EATFILED debug('Parsing XML data for EATFILEE') eData = base64.decodestring(root.find(CAST_OUT_NAME_TAG).text) #Data for EATFILEE is just one field if 'FauxLinenElec.eatvrp' in eData: eData = 'G:\\EAT\\Parameter\\Cast-out\\1200.eatvrp' debug('Detected FauxLinenElec.eatvrp as cast out file. Writing 1200.eatvrp to database instead...') debug('\t\tCast Out File(Hooks): ' + eData) debug('Parsing XML data for EATFILEF') fData = getFileFData(root) #Get data to insert into EATFILEF debug('Parsing XML data for EATFILEG') try: gData = getFileGData(root) #Get data to insert into EATFILEG except RuntimeError, e: #Data base fields too small to accommodate numbers larger than 999, display a message for now debugError(e) JOptionPane.showMessageDialog(frame, aData[0] + '.xml has card counts over 999!\n\rYou\'ll have to key this one by hand for now :(\n\rHit okay to resume processing...', "Too Many Cards!", JOptionPane.ERROR_MESSAGE) return debug('--------------------------------') #File A debug('Preparing SQL statement for EATFILEA') fileAInsert.setObject(1, aData[0]) fileAInsert.setObject(2, aData[1]) fileAInsert.setObject(3, aData[0]) fileAInsert.setObject(4, aData[2]) fileAInsert.setObject(5, aData[3]) fileAInsert.setObject(6, aData[4]) fileAInsert.setObject(7, aData[5]) fileAInsert.setObject(8, aData[6]) #File B debug('Preparing SQL statement for EATFILEB') fileBInsert.setObject(1, patternNumber) fileBInsert.setObject(2, applicationBoxMotionSid) fileBInsert.setObject(3, bData) #File C debug('Preparing SQL batch for EATFILEC') for row in cData: fileCInsert.setObject(1, row[0]) fileCInsert.setObject(2, row[1]) fileCInsert.setObject(3, row[2]) fileCInsert.setObject(4, row[3]) fileCInsert.addBatch() #File D debug('Preparing SQL batch for EATFILED') for row in dData: fileDInsert.setObject(1, row[0]) fileDInsert.setObject(2, row[1]) fileDInsert.setObject(3, row[2]) fileDInsert.setObject(4, row[3]) fileDInsert.setObject(5, row[4]) fileDInsert.setObject(6, row[5]) fileDInsert.addBatch() #File E debug('Preparing SQL statement for EATFILEE') fileEInsert.setObject(1, patternNumber) fileEInsert.setObject(2, applicationMachineSid) fileEInsert.setObject(3, eData) #File F debug('Preparing SQL batch for EATFILEF') for row in fData: fileFInsert.setObject(1, row[0]) fileFInsert.setObject(2, row[1]) fileFInsert.setObject(3, row[2]) fileFInsert.setObject(4, row[3]) fileFInsert.addBatch() #File G debug('Preparing SQL batch for EATFILEG') for row in gData: fileGInsert.setObject(1, row[0]) fileGInsert.setObject(2, row[1]) fileGInsert.setObject(3, row[2]) fileGInsert.setObject(4, row[3]) fileGInsert.setObject(5, row[4]) fileGInsert.addBatch() #Do not alter stats file when testing if not (DEBUG_MODE): statsInsert.setObject(1, datetime.datetime.now().strftime('%c')[:24]) statsInsert.setObject(2, aData[0][:7]) #The pattern number isn't really vital; if for some reason one comes through w/ more than 7 characters just slice them off debug('--------------------------------') debug('Inserting data into EATFILEA') fileAInsert.executeUpdate() debug('Inserting data into EATFILEB') fileBInsert.executeUpdate() debug('Inserting data into EATFILEC') fileCInsert.executeBatch() debug('Inserting data into EATFILED') fileDInsert.executeBatch() debug('Inserting data into EATFILEE') fileEInsert.executeUpdate() debug('Inserting data into EATFILEF') fileFInsert.executeBatch() debug('Inserting data into EATFILEG') fileGInsert.executeBatch() #Don't touch the stats table if in debug mode if not (DEBUG_MODE): debug('Updating Statistics File') statsInsert.executeUpdate() ''' @param root: An elementtree.Element object referring to the root tag of the XML document Most of the files that are being updated on the AS400 require the Sid of the containing XML tag for some reason. From what I can tell the Sid exists because there are several top level tags that are all named 'Application' this is not a problem however because they all have a different Sid. Since we know the names of the child tags that contain the data we are after we can loop through the top level tags and check their children for the appropriate tag names. When one is found that is used a relevant variable is set to the Sid of the parent tag. It seems to me like these will never change and I could have just made them constants. Might as well play it safe right? ''' def getSids(root): global transferObjectSid global applicationBoxMotionSid global applicationMachineSid global applicationCalcJCSimuSid #Get sid values for the TransferObject tag, and the 3 Application tags that contain AddBoxMotion, MachineDefinition, and CalcJCSimu #ForLoopCeption for child in root: if(child.tag == 'Entries'): for subChild in child: if(subChild.tag == 'TransferObject'): transferObjectSid = subChild.get('sid') elif(subChild.tag == 'Application'): for subSubChild in subChild: if(subSubChild.tag == 'AddBoxMotion'): applicationBoxMotionSid = subChild.get('sid') elif(subSubChild.tag == 'MachineDefinition'): applicationMachineSid = subChild.get('sid') elif(subSubChild.tag == 'CalcJCSimu'): applicationCalcJCSimuSid = subChild.get('sid') ''' @param root: An elementtree.Element object referring to the root tag of the XML document This method parses the data required for EATFILEA on the AS400 This data includes: Pattern Number Picks Last Saved Time Horizontal Warp Ends Vertical Repeat The data is returned in a list: data = [patternNumber, Picks, Last_Saved_Time, Horizontal_Warp_Ends, Vertical_Repeat] ''' def getFileAData(root): global patternNumber ret = [] ret.append(base64.decodestring(root.find(PATTERN_NUMBER_TAG).text)) ret.append(transferObjectSid) ret.append(root.find(PICKS_TAG).text) x = base64.decodestring(root.find(TIME_STAMP_TAG_A).text).split(' ') ret.append(x[0] + x[1]) ret.append(root.find(HORIZ_WARP_ENDS_TAG).text) ret.append(root.find(VERT_REPEAT_TAG).text) debug('\t\tPattern Number: ' + ret[0]) debug('\t\tSid: ' + ret[1]) debug('\t\tPicks: ' + ret[2]) debug('\t\tTime Stamp: ' + ret[3]) debug('\t\tHorizontal Warp Ends: ' + ret[4]) debug('\t\tVertical Repeat: ' + ret[5]) try: ret.append(base64.decodestring(root.find(NOTE_TAG).text)) #This note is almost never populated, none-the-less we will try.. except AttributeError: ret.append(' ') debug('\t\tNote: ' + ret[6]) patternNumber = ret[0] return ret ''' @param root: An elementtree.Element object referring to the root tag of the XML document This method parses the data required for EATFILEA on the AS400 This data includes: Pattern Number Sid Number of the containing XML tag Sequence number Card Count, these always seem to be the same values as Ground Cards in EATFILEG After looking through the programs on the AS400 that interpret the data that is sent; I've found that only files A, B, E, F, and G are actually used. C and D just seem to be cleared. As a result of this, I've made it so that if the XML tags for the data are not found in the file it is just ignored instead of halting processing. 02/18/14 - 6 months later; haven't run into any problems ignoring files C and D if no data is located. patternNumber must have been previously determined by getFileAData()! The appropriate Sid number must have been previously determined by getSids()! The data is returned in a 2D list: data = [[patternNumber, Sid, Sequence, Card_Count], [patternNumber, Sid, Sequence, Card_Count], [patternNumber, Sid, Sequence, Card_Count]] ''' def getFileCData(root): debug('\tPattern\tSID\tSequence\tCard Count') ret = [] boxMotionColorList = root.find(SEQUENCE_NUM_TAG_C) try: #This data seems inconsequential to the process, if boxMotionColorList ends up None we'll return an empty list instead of halting processing for listEntry in boxMotionColorList: idno = listEntry.get('ID') cardCount =listEntry.find(CARD_COUNT_TAG).text row = [patternNumber, applicationBoxMotionSid, idno, cardCount] ret.append(row) debug('\t' + row[0] + '\t' + row[1] + '\t' + row[2] + '\t\t' + row[3]) return ret except TypeError: debug('\tWarning: No data was located for EATFILEC. You can probably ignore this.') return [] except JavaException, e: debug('Encountered problems while parsing data for file C. You can probably ignore this.') debug('Error Message: ' + e.toString()) return [] ''' @param root: An elementtree.Element object referring to the root tag of the XML document This method parses the data required for EATFILED on the AS400 This data includes: Pattern Number Sid Number of the containing XML tag Sequence Number Artist Computer Name Time Stamp After looking through the programs on the AS400 that interpret the data that is sent; I've found that only files A, B, E, F, and G are actually used. C and D just seem to be cleared. As a result of this, I've made it so that if the XML tags for the data are not found in the file it is just ignored instead of halting processing. 02/18/14 - 6 months later; haven't run into any problems ignoring files C and D if no data is located. patternNumber must have been previously determined by getFileAData()! The appropriate Sid number must have been previously determined by getSids()! The data is returned in a 2D list: data = [[patternNumber, Sid, Sequence, Artist, Computer, Time_Stamp], [patternNumber, Sid, Sequence, Artist, Computer, Time_Stamp], [patternNumber, Sid, Sequence, Artist, Computer, Time_Stamp]] ''' def getFileDData(root): debug('\tPattern\tSID\tSequence\tArtist\t\tComputer\t\t\tTime Stamp') ret = [] modifcationEntries = root.find(SEQUENCE_NUM_TAG_D) try: #This data seems inconsequential to the process, if modifcationEntries ends up None we'll return an empty list instead of halting processing for listEntry in modifcationEntries: idno = listEntry.get('ID') artist = base64.decodestring(listEntry.find(ARTIST_TAG).text) computer = base64.decodestring(listEntry.find(COMPUTER_NAME_TAG).text) timestamp = base64.decodestring(listEntry.find(TIME_STAMP_TAG_D).text).split(' ') timestamp = timestamp[0] + timestamp[1] row = [patternNumber, transferObjectSid, idno, artist, computer, timestamp] ret.append(row) debug('\t' + row[0] + '\t' + row[1] + '\t' + row[2] + '\t\t' + row[3] + '\t' + row[4] + '\t\t\t' + row[5]) except TypeError: debug('\tWarning: No data was located for EATFILED. You can probably ignore this.') except JavaException: debug('\tWarning: Modification history corrupt. You can probably ignore this.') return ret ''' @param root: An elementtree.Element object referring to the root tag of the XML document This method parses the data required for EATFILEF on the AS400 This data includes: Pattern Number Sid Number of the containing XML tag Yarn Sequence Number Yarn Type and Yarn Color in one field patternNumber must have been previously determined by getFileAData()! The appropriate Sid number must have been previously determined by getSids()! The data is returned in a 2D list: data = [[patternNumber, Sid, Sequence, Yarn], [patternNumber, Sid, Sequence, Yarn], [patternNumber, Sid, Sequence, Yarn]] ''' def getFileFData(root): debug('\tPattern\tSid\tSequence\tYarn') ret = [] weftYarnList = root.find(SEQUENCE_NUM_TAG_F) for listEntry in weftYarnList: idno = listEntry.get('ID') yarn = base64.decodestring(listEntry.find(PAT_NUM_TAG_F).text) row = [patternNumber, applicationCalcJCSimuSid, idno, yarn] ret.append(row) debug('\t' + row[0] + '\t' + row[1] + '\t' + row[2] + '\t\t' + row[3]) return ret ''' @bug: Due to the inadequate size of the card fields (3 digits) we are unable to process patterns that have more than 999 cards for a single position. @param root: An elementtree.Element object referring to the root tag of the XML document This method parses the data required for EATFILEG on the AS400 This data includes: Pattern Number Sid Number of the containing XML tag Yarn Sequence Number Ground Cards Tissue Cards patternNumber must have been previously determined by getFileAData()! The appropriate Sid number must have been previously determined by getSids()! The data is returned in a 2D list: data = [[patternNumber, Sid, Sequence, Ground_Cards, Tissue_Cards], [patternNumber, Sid, Sequence, Ground_Cards, Tissue_Cards], [patternNumber, Sid, Sequence, Ground_Cards, Tissue_Cards]] ''' def getFileGData(root): debug('\tPattern\tSid\tSequence\tGround Cards\tTissue Cards') ret = [] yarnNums = root.findall(YARN_NUM_TAG) tissueCards = root.findall(TISSUE_CARD_TAG) groundCards = root.findall(GROUND_CARD_TAG) for i in range(0,len(yarnNums)): row = [patternNumber, applicationBoxMotionSid, yarnNums[i].text, groundCards[i].text, tissueCards[i].text] ret.append(row) debug('\t' + row[0] + '\t' + row[1] + '\t' + row[2] + '\t\t' + row[3] + '\t\t' + row[4]) if(int(row[3]) > 999) or (int(row[4]) > 999): raise RuntimeError('ERROR: Card counts are too large for the fields on the AS400 (> 999). You\'ll have to key this one in by hand. Sorry!') return ret ''' XSD files contain schema information for XML files. They can be used to validate the structure of an associated XML file. We do not use them for anything so this method is called during startup to clear them out of the folder. They are generated by the EAT software when the designers export a pattern in XML format. ''' def deleteXSDFiles(): for f in glob.glob(NETWORK_PATH + "\*.xsd"): #Generate a list of all XSD files in the network folder os.remove(f) #Delete each XSD file ''' Delete log files after KEEP_LOG_FILES days. Called on startup ''' def pruneLogFiles(): filelist = glob.glob(NETWORK_PATH + "\\logs\\" + LOG_FILE_REGEX) #Generate a list of all text files in the log folder then = (datetime.datetime.now() - timedelta(days=KEEP_LOG_FILES)) #logs older than this will be deleted for f in filelist: try: #Super dirty hack, don't care. The worst that can happen is that some logs won't get deleted ymd = f.split('\\')[LOG_FOLDER_DEPTH].split('.')[0].split('-') now = datetime.datetime(int(ymd[2]), int(ymd[0]), int(ymd[1])) if(now < then): os.remove(f) except ValueError: pass ''' @param qtm: Builds a list of files to be parsed and uploaded to the AS400 Loops through every row in the table, if the process boolean is set the full path is prepended to the file name and added to the list that is returned by the method. ''' def getFilesSelectedForProcessing(qtm): ret = [] for row in qtm.cache: if(row[1]): ret.append(NETWORK_PATH + row[0]) return ret ''' This method is the actionPerformed property for the 'Process!' button. Everything not having to do with the UI happens starting from here. 1. The log file is opened and a dashed separator line is written to help distinguish sessions in the log file 2. getFilesSelectedForProcessing() is called to create a list of files that were selected in the table by the user 3. prepareToSend() is called which opens a connection with the AS400 and pre-compiles all of the SQL statements 4. parseAndSend() is called for each file in the list. This is where most of the processing occurs. 5. QCMDEXC is invoked via a stored procedure call in order to execute STMLIB/XMLPARSE. This causes the data uploaded to be processed by the AS400 right away. 6. The open connection with the AS400 is closed 7. The log file buffer is flushed in the finally block, to make sure any error information gets written to the log file. ''' def process(event): global logFile global currentlyProcessing try: frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)) #Change mouse icon to loading icon for duration of this method logFile = openLogFile() logFile.write('<--------------------------EAT DATA UPLOAD REQUESTED-------------------------->\n') debug('Sunbury Textile Mills - XMLMapper v' + VERSION_ID + ' - Author: Wesley Ray') files = getFilesSelectedForProcessing(qtm) debug(getpass.getuser() + ' selected ' + str(len(files)) + ' file(s) for processing') for f in files: debug('\t\t' + f) prepareToSend() #Open connection with server, pre-compile SQL statements for inFile in files: currentlyProcessing = inFile.split('\\')[4] parseAndSend(inFile) #Parse XML data and upload to server for each file qtm.setValueAt(False, qtm.cache.index([currentlyProcessing, True]), 1) #Un-check successfully processed files debug(' ') debug('Processing completed normally! AS400 connection closed.') JOptionPane.showMessageDialog(None, 'Processing of ' + str(len(files)) + ' file(s) completed with no errors reported!', 'Success!', JOptionPane.INFORMATION_MESSAGE) except Exception, e: frame.setCursor(Cursor.getDefaultCursor()) debug('An error has occurred while processing ' + currentlyProcessing) debug('Only the files processed successfully up to this point have been sent to the AS400!') debugError(e) JOptionPane.showMessageDialog(frame, 'An error occurred while processing ' + currentlyProcessing + '\n\rProbably a bogus XML document from the designer, have them re-send it.\n\rIf the problem persists call Wes @ Ext. 234', "An Error Has Occurred!", JOptionPane.ERROR_MESSAGE) except JavaException, e: frame.setCursor(Cursor.getDefaultCursor()) debug('An error has occurred while processing ' + currentlyProcessing) debug('Only the files processed successfully up to this point have been sent to the AS400!') debug('Error Message: ' + e.toString()) JOptionPane.showMessageDialog(frame, 'An error occurred while processing ' + currentlyProcessing + '\n\rProbably a bogus XML document from the designer, have them re-send it.\n\rIf the problem persists call Wes @ Ext. 234', "An Error Has Occurred!", JOptionPane.ERROR_MESSAGE) finally: fileAInsert.close() #Close PreparedStatement Objects fileBInsert.close() fileCInsert.close() fileDInsert.close() fileEInsert.close() fileFInsert.close() fileGInsert.close() try: blah = connection.createStatement().executeQuery('select count(*) from wes.xmlstats') blah.next() blah = blah.getObject(1) debug(str(blah) + ' files processed since 05/15/14!') debug('Attempting to force data update on AS400...') forceUpdate.execute() debug('Force update successful! The printouts should be on P7 now.') except JavaException, e: debug('Force Update Error: ' + e.toString()) JOptionPane.showMessageDialog(frame, 'Could not force update!\n\rThe data should be picked up automatically in about 5 minutes.', JOptionPane.ERROR_MESSAGE) forceUpdate.close() #Close CallableStatement object connection.close() #Close Connection Object logFile.flush() qtm.fireTableChanged(None) #Repaint Table frame.setCursor(Cursor.getDefaultCursor()) ''' This method is the actionPerformed property for both the 'Select' and 'Deselect' buttons. Basically loops through each row in the table data and sets the process boolean equal to what (event.getSource() is selectButton) evaluates to; which will be True if select all was clicked and false if deselect all was clicked. ''' def selectAll(event): for row in qtm.cache: row[1] = (event.getSource() is selectButton) #Set 'process?' boolean for each row in the table qtm.fireTableChanged(None) #Repaint Table ''' I found out that Kathy will typically not delete the XML files directly after sending them, rather, she keeps them until the design process is finished so that if there is an issue she doesn't need to have the designers re-send the XML. So I got rid of the delete prompt in the process method and added a button to 'Delete Selected' files. All this does is get the selected file list from the file table and delete them from the network folder. ''' def deleteSelected(event): global logFile logFile = openLogFile() dialogResult = JOptionPane.showConfirmDialog(frame, 'Are you sure you would like to delete the selected files? This _CANNOT_ be undone!', 'Delete Files', JOptionPane.YES_NO_OPTION) if(dialogResult == JOptionPane.YES_OPTION): for f in getFilesSelectedForProcessing(qtm): #Fetch list of selected files from the file table os.remove(f) #Delete them debug(f + ' Deleted!') #Log this qtm.loadTable(glob.glob(NETWORK_PATH + "\*.xml")) #Refresh file table logFile.flush() logFile.close() def refreshList(event): qtm.loadTable(glob.glob(NETWORK_PATH + "\*.xml")) #Call the loadTable method with a list of XML files in the network folder if __name__ == '__main__': for line in ASCII_ART: print line deleteXSDFiles() #Delete all XSD files in the network folder pruneLogFiles() #Delete log files older than 7 days frame = JFrame('Sunbury Textile Mills - XML Mapper') #Create a new JFrame() frame.setIconImage(ImageIcon('icon.png').getImage()) #Found an XML icon on the interwebs, better than the java logo anyways... UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName()) #Set look-and-feel to Windows-ish frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE) #Call System.exit on frame closing frame.setSize(340, 340) #Good dimensions as determined by me frame.setLayout(BorderLayout()) #Pretty simple GUI, we'll use BorderLayout() table = JTable() #Create file table table.setModel(FileTableModel()) #Load my AbstractTableModel implementation I created for this program scrollPane = JScrollPane(table) #Create a JScrollPane to put the table in table.setFillsViewportHeight(True) #Make table always fill the scrollPane table.setRowSelectionAllowed(False) #Disallow row selection, we are using in-cell JCheckBoxes instead qtm = table.getModel() #Get a reference to the TabelModel so we can access the table data and loadTable() method qtm.loadTable(glob.glob(NETWORK_PATH + "\*.xml")) #Call the loadTable method with a list of XML files in the network folder processButton = JButton('Process!', actionPerformed=process) #JButton to initiate EAT data upload selectButton = JButton('Select All', actionPerformed=selectAll) #JButton to select all files in the list deselectButton = JButton('Deselect All', actionPerformed=selectAll) #JButton to deselect all selected files in the list refreshButton = JButton('', actionPerformed=refreshList) refreshButton.setIcon(ImageIcon(ImageIO.read(File('refresh.png')))) refreshButton.setToolTipText('Refresh file list. Useful for when designers save files to the folder after you start this program.') deleteButton = JButton('', actionPerformed=deleteSelected) deleteButton.setIcon(ImageIcon(ImageIO.read(File('delete.png')))) deleteButton.setToolTipText('Delete selected files') topPanel = JPanel() #Since we are using BorderLayout, create a JPanel to contain all of the JButtons topPanel.add(selectButton) topPanel.add(deselectButton) topPanel.add(refreshButton) topPanel.add(deleteButton) frame.add(topPanel, BorderLayout.NORTH) #Add the button JPanel to BorderLayout.NORTH frame.add(scrollPane, BorderLayout.CENTER) #Add the JScrollPane to BorderLayout.CENTER frame.add(processButton, BorderLayout.SOUTH) #Add the 'Process!' JButton to BorderLayout.SOUTH #frame.pack() #JTable's behave strangely with this it seems, I figured out good dimensions myself frame.setLocationRelativeTo(None) #Center the frame on screen frame.setResizable(False) #Do not allow resizing the frame frame.setVisible(True) #Display the frame