diff --git a/EATXMLmapper/STM/Mapper.py b/EATXMLmapper/STM/Mapper.py new file mode 100644 index 0000000..c69a61e --- /dev/null +++ b/EATXMLmapper/STM/Mapper.py @@ -0,0 +1,805 @@ +''' +Created on Sep 10, 2013 + +@author: Wesley Ray + +Replacement for the XMLmapper software purchased a few years ago. The current one has too many driver issues/conflicts and +I knew that parsing XML with Python is pretty easy. + +This makes use of xml.etree.ElementTree to create a Document Object Model (DOM) of the input files. This makes it very easy to +locate information if you know the the tag structure; which we do from the .ini file of the old XMLmapper program. Lucky that +programmer decided against hard-coding that information into the binaries. + +Taking the DOM approach is inherently slow (the entire file is loaded into memory and parsed). It's not a major problem for our +purposes as the XML files are < 10MB, it still does take a moment to load the XML file initially. The parsing and uploading of +the data afterwards is almost instantaneous however. +''' + +from STM.TabelModel import FileTableModel +from com.ibm.as400.access import AS400JDBCDriver +from java.awt import BorderLayout, Cursor +from java.io import ByteArrayInputStream, File +from java.sql import DriverManager +from javax.imageio import ImageIO +from javax.swing import JFrame, UIManager, JTable, JScrollPane, JButton, JPanel, \ + JOptionPane, ImageIcon +import base64 +import datetime +from datetime import timedelta +import glob +import java.lang.Exception as JavaException +import os, getpass +import xml.etree.ElementTree as ET + +VERSION_ID = '1.6' +NETWORK_PATH = '\\\\EATSRV\\XML Files' +LOG_FOLDER_DEPTH = NETWORK_PATH.count('\\') + 2; +LOG_FILE_REGEX = '[0-9][0-9]-[0-9][0-9]-[0-9][0-9][0-9][0-9].txt' +KEEP_LOG_FILES = 7 #Number of days to keep log files in the log folder +DEBUG_MODE = True #Set this True to write to test database and debug log file + +ASCII_ART = [' .d8888b. 88888888888 888b d888', + ' d88P Y88b 888 8888b d8888', + ' Y88b. 888 88888b.d88888', + ' "Y888b. 888 888Y88888P888', + ' "Y88b. 888 888 Y888P 888', + ' "888 888 888 Y8P 888', + ' Y88b d88P 888 888 " 888', + ' "Y8888P" 888 888 888'] + +''' +Invoke QCMDEXC via a stored procedure (Thanks IBM!) so that we can get the EAT data into the system right away. + +This runs a CL program I've written on the AS400 called EATPARSE. +The source code for EATPARSE is located in STMLIB/QCLLESRC +''' +FORCE_PROCESSING = "call qsys.qcmdexc('CALL STMLIB/EATPARSE',0000000020.00000)" + +#EATFILEA +PATTERN_NUMBER_TAG = 'Entries/TransferObject/PseudoColorImageTO/Name' # Maps to EATDATA/EATFILEA.EATNAM +PICKS_TAG = 'Entries/TransferObject/PseudoColorImageTO/WeftDensity' # Maps to EATDATA/EATFILEA.EATPKS +TIME_STAMP_TAG_A = 'Entries/TransferObject/PseudoColorImageTO/LastSavedTime' # Maps to EATDATA/EATFILEA.EATTMS +HORIZ_WARP_ENDS_TAG = 'Entries/TransferObject/PseudoColorImageTO/SizeX' # Maps to EATDATA/EATFILEA.EATHWE +VERT_REPEAT_TAG = 'Entries/TransferObject/PseudoColorImageTO/WorldSizeY' # Maps to EATDATA/EATFILEA.EATVRP +NOTE_TAG = 'Entries/TransferObject/PseudoColorImageTO/Note' # Maps to EATDATA/EATFILEA.EATNOT + +#EATFILEB +TOTAL_CARDS_TAG = 'Entries/Application/AddBoxMotion/BoxMotionInfo/BoxMotionTotalPicks' # Maps to EATDATA/EATFILEB.EATCDS + +#EATFILEC +#Children of BoxMotionColorList are ListEntry tags +SEQUENCE_NUM_TAG_C = 'Entries/Application/AddBoxMotion/BoxMotionInfo/BoxMotionList/ListEntry/BoxMotionListEntry/BoxMotionColorList' # Maps to EATDATA/EATFILEC.EATSEQ +#ListEntry Tags have these as children +CARD_COUNT_TAG = 'BoxMotionColorListEntry/ColorCount' # Maps to EATDATA/EATFILEC.EATCCT + +#EATFILED +SEQUENCE_NUM_TAG_D = 'Entries/TransferObject/PseudoColorImageTO/ModificationHistoryList' # Maps to EATDATA/EATFILED.EATSQ5 +ARTIST_TAG = 'ModificationHistoryListEntry/Artist' # Maps to EATDATA/EATFILED.EATART +COMPUTER_NAME_TAG = 'ModificationHistoryListEntry/ComputerName' # Maps to EATDATA/EATFILED.EATCNM +TIME_STAMP_TAG_D = 'ModificationHistoryListEntry/Time' # Maps to EATDATA/EATFILED.EATTMS + +#EATFILEE +CAST_OUT_NAME_TAG = 'Entries/Application/MachineDefinition/MachDefCastOutName' # Maps to EATDATA/EATFILEE.EATDON + +#EATFILEF +SEQUENCE_NUM_TAG_F = 'Entries/Application/CalcJCSimu/SimuYarnList/WeftYarnList' # Maps to EATDATA/EATFILEF.EATSEQ +PAT_NUM_TAG_F = 'YarnName' # Maps to EATDATA/EATFILEF.EATNAM + +#EATFILEG +YARN_NUM_TAG = 'Entries/Application/AddBoxMotion/BoxMotionInfo/BoxMotionYarnList/ListEntry/BoxMotionYarnListEntry/BoxMotionYarnNumber' # Maps to EATDATA/EATFILEG.EATYRN +TISSUE_CARD_TAG = 'Entries/Application/AddBoxMotion/BoxMotionInfo/BoxMotionYarnList/ListEntry/BoxMotionYarnListEntry/BoxMotionYarnCountRegSingle' # Maps to EATDATA/EATFILEG.EATTCD +GROUND_CARD_TAG = 'Entries/Application/AddBoxMotion/BoxMotionInfo/BoxMotionYarnList/ListEntry/BoxMotionYarnListEntry/BoxMotionYarnCountSingle' # Maps to EATDATA/EATFILEG.EATGCD + +if(DEBUG_MODE): + #SQL statements + FILE_A_INSERT = 'insert into WES.EATFILEA values(?, ?, ?, ?, ?, ?, ?, ?)' + FILE_B_INSERT = 'insert into WES.EATFILEB values(?, ?, ?)' + FILE_C_INSERT = 'insert into WES.EATFILEC values(?, ?, ?, ?)' + FILE_D_INSERT = 'insert into WES.EATFILED values(?, ?, ?, ?, ?, ?)' + FILE_E_INSERT = 'insert into WES.EATFILEE values(?, ?, ?)' + FILE_F_INSERT = 'insert into WES.EATFILEF values(?, ?, ?, ?)' + FILE_G_INSERT = 'insert into WES.EATFILEG values(?, ?, ?, ?, ?)' +else: + #SQL statements + FILE_A_INSERT = 'insert into EATDATA.EATFILEA values(?, ?, ?, ?, ?, ?, ?, ?)' + FILE_B_INSERT = 'insert into EATDATA.EATFILEB values(?, ?, ?)' + FILE_C_INSERT = 'insert into EATDATA.EATFILEC values(?, ?, ?, ?)' + FILE_D_INSERT = 'insert into EATDATA.EATFILED values(?, ?, ?, ?, ?, ?)' + FILE_E_INSERT = 'insert into EATDATA.EATFILEE values(?, ?, ?)' + FILE_F_INSERT = 'insert into EATDATA.EATFILEF values(?, ?, ?, ?)' + FILE_G_INSERT = 'insert into EATDATA.EATFILEG values(?, ?, ?, ?, ?)' + +STATS_INSERT = 'insert into WES.XMLSTATS values(?, ?)' + +#Global Vars +transferObjectSid = 0 +applicationBoxMotionSid = 0 +applicationMachineSid = 0 +applicationCalcJCSimuSid = 0 +patternNumber = '' +connection = None +fileAInsert = None +fileBInsert = None +fileCInsert = None +fileDInsert = None +fileEInsert = None +fileFInsert = None +fileGInsert = None +statsInsert = None +forceUpdate = None +setStats = None +frame = None +qtm = None +logFile = None +currentlyProcessing = '' + +''' +Returns a BufferedImage object of the preview image of the pattern encoded in the XML + +Currently not used for anything, I wanted to make a nifty progress dialog that displayed a thumbnail of the pattern whose XML is currently +being processed +''' +def getPreviewImage(root): + return ImageIO.read(ByteArrayInputStream(base64.decodestring(root.find('Entries/TransferObject/PseudoColorImageTO/Preview').text))) + +''' +This method is one of the reasons why I love Python. In one line we are opening (or creating if the file doesn't exist) a +text file with the current date as the name. + +Example: 10-27-13.txt +''' +def openLogFile(): + if(DEBUG_MODE): + return open(datetime.datetime.now().strftime(NETWORK_PATH + '\\logs\\%m-%d-%Y_DEBUG.txt'), 'a') # The 'a' signifies append mode, file can only be written to. Does not affect current contents of file. + else: + return open(datetime.datetime.now().strftime(NETWORK_PATH + '\\logs\\%m-%d-%Y.txt'), 'a') + +''' +This method simply prepends a time stamp to a string you wish to print to stdout. +It also writes the resulting string out to the log file +''' +def debug(msg): + now = datetime.datetime.now() + if(now.minute < 10): + mins = '0' + str(now.minute) + else: + mins = str(now.minute) + msg = '[' + str(now.hour) + ':' + mins + '] ' + msg + print msg + logFile.write(msg + '\n') + +''' +Modified version of debug that takes a Python Error as a parameter so that it can be +printed to the stdout and written to the log file +''' +def debugError(err): + now = datetime.datetime.now() + if(now.minute < 10): + mins = '0' + str(now.minute) + else: + mins = str(now.minute) + msg = '[' + str(now.hour) + ':' + mins + '] Error Message:' + print msg, err + logFile.write(msg + repr(err) + '\n') + +''' +Loads the IBM AS400 JDBC driver and opens a connection with the AS400 +''' +def connect(): + DriverManager.registerDriver(AS400JDBCDriver()) + return DriverManager.getConnection('jdbc:as400://SUNBURY', 'USER', 'PASSWORD') + +''' +Opens a connection to the AS400, global var 'connection' gets the reference to the connection object +pre-compiles SQL statements and assigns their references to the appropriate global variables +''' +def prepareToSend(): + global connection + global fileAInsert + global fileBInsert + global fileCInsert + global fileDInsert + global fileEInsert + global fileFInsert + global fileGInsert + global forceUpdate + global statsInsert + debug('Connecting to AS400 as user: EAT') + connection = connect() #Open a connection + debug('Pre-compiling SQL statements') + fileAInsert = connection.prepareStatement(FILE_A_INSERT) + fileBInsert = connection.prepareStatement(FILE_B_INSERT) + fileCInsert = connection.prepareStatement(FILE_C_INSERT) + fileDInsert = connection.prepareStatement(FILE_D_INSERT) + fileEInsert = connection.prepareStatement(FILE_E_INSERT) + fileFInsert = connection.prepareStatement(FILE_F_INSERT) + fileGInsert = connection.prepareStatement(FILE_G_INSERT) + statsInsert = connection.prepareStatement(STATS_INSERT) + forceUpdate = connection.prepareCall(FORCE_PROCESSING) + +''' +This method does the actual load, parse, upload routine. Nothing profound here see the in-line comments in the method. + +[MODIFIED] 9/13/13 - It was bugging me the whole time that if a SQLException was thrown all operations for subsequent files + would not occur; resulting in data being in some of the files (maybe a,b,c) and not in the rest which + would likely be an issue for the job that picks up the data on the AS400. + So I found out that java.sql.PreparedStatement has addBatch() and executeBatch() methods. So now all + of the actual inserting can be done after the prepared statements are created. It is the setting of the parameters + of the prepared statements that is most likely to throw an exception. If that occurs now, none of the files + will be updated, instead of just the files leading up to the exception being updated. + +[MODIFIED] 9/17/13 - Wow.. I have seen the light. PreparedStatement.setObject() will do type conversion for me... + No more type conversions, and setString, setInt, setYourFace... + On a more serious note, this did work well and it really cleans up the code. This was easy to do partially + because all of the data parsed from the XML files are strings. Strings can certainly be converted to + integers and floats as long as they actually contain numbers. You would need to be careful in a language + like Python to know what type of data is contained in a variable when passing it to setObject or you'll + get SQLExceptions. +''' +def parseAndSend(inFile): + debug('Loading ' + inFile + '...') + tree = ET.parse(inFile) + debug('--------------------------------') + root = tree.getroot() #Get root XML tag + + getSids(root) #Parse sid's into global vars + debug('Parsing XML data for EATFILEA') + aData = getFileAData(root) #Get data to insert into EATFILEA + + debug('Parsing XML data for EATFILEB') + bData = root.find(TOTAL_CARDS_TAG).text #Data for EATFILEB is just one field + debug('\t\tTotal Cards: ' + bData) + + debug('Parsing XML data for EATFILEC') + cData = getFileCData(root) #Get data to insert into EATFILEC + + debug('Parsing XML data for EATFILED') + dData = getFileDData(root) #Get data to insert into EATFILED + + debug('Parsing XML data for EATFILEE') + eData = base64.decodestring(root.find(CAST_OUT_NAME_TAG).text) #Data for EATFILEE is just one field + if 'FauxLinenElec.eatvrp' in eData: + eData = 'G:\\EAT\\Parameter\\Cast-out\\1200.eatvrp' + debug('Detected FauxLinenElec.eatvrp as cast out file. Writing 1200.eatvrp to database instead...') + debug('\t\tCast Out File(Hooks): ' + eData) + + debug('Parsing XML data for EATFILEF') + fData = getFileFData(root) #Get data to insert into EATFILEF + + debug('Parsing XML data for EATFILEG') + try: + gData = getFileGData(root) #Get data to insert into EATFILEG + except RuntimeError, e: #Data base fields too small to accommodate numbers larger than 999, display a message for now + debugError(e) + JOptionPane.showMessageDialog(frame, aData[0] + '.xml has card counts over 999!\n\rYou\'ll have to key this one by hand for now :(\n\rHit okay to resume processing...', "Too Many Cards!", JOptionPane.ERROR_MESSAGE) + return + + debug('--------------------------------') + + #File A + debug('Preparing SQL statement for EATFILEA') + fileAInsert.setObject(1, aData[0]) + fileAInsert.setObject(2, aData[1]) + fileAInsert.setObject(3, aData[0]) + fileAInsert.setObject(4, aData[2]) + fileAInsert.setObject(5, aData[3]) + fileAInsert.setObject(6, aData[4]) + fileAInsert.setObject(7, aData[5]) + fileAInsert.setObject(8, aData[6]) + + #File B + debug('Preparing SQL statement for EATFILEB') + fileBInsert.setObject(1, patternNumber) + fileBInsert.setObject(2, applicationBoxMotionSid) + fileBInsert.setObject(3, bData) + + #File C + debug('Preparing SQL batch for EATFILEC') + for row in cData: + fileCInsert.setObject(1, row[0]) + fileCInsert.setObject(2, row[1]) + fileCInsert.setObject(3, row[2]) + fileCInsert.setObject(4, row[3]) + fileCInsert.addBatch() + + #File D + debug('Preparing SQL batch for EATFILED') + for row in dData: + fileDInsert.setObject(1, row[0]) + fileDInsert.setObject(2, row[1]) + fileDInsert.setObject(3, row[2]) + fileDInsert.setObject(4, row[3]) + fileDInsert.setObject(5, row[4]) + fileDInsert.setObject(6, row[5]) + fileDInsert.addBatch() + + #File E + debug('Preparing SQL statement for EATFILEE') + fileEInsert.setObject(1, patternNumber) + fileEInsert.setObject(2, applicationMachineSid) + fileEInsert.setObject(3, eData) + + #File F + debug('Preparing SQL batch for EATFILEF') + for row in fData: + fileFInsert.setObject(1, row[0]) + fileFInsert.setObject(2, row[1]) + fileFInsert.setObject(3, row[2]) + fileFInsert.setObject(4, row[3]) + fileFInsert.addBatch() + + #File G + debug('Preparing SQL batch for EATFILEG') + for row in gData: + fileGInsert.setObject(1, row[0]) + fileGInsert.setObject(2, row[1]) + fileGInsert.setObject(3, row[2]) + fileGInsert.setObject(4, row[3]) + fileGInsert.setObject(5, row[4]) + fileGInsert.addBatch() + + #Do not alter stats file when testing + if not (DEBUG_MODE): + statsInsert.setObject(1, datetime.datetime.now().strftime('%c')[:24]) + statsInsert.setObject(2, aData[0][:7]) #The pattern number isn't really vital; if for some reason one comes through w/ more than 7 characters just slice them off + + debug('--------------------------------') + + debug('Inserting data into EATFILEA') + fileAInsert.executeUpdate() + + debug('Inserting data into EATFILEB') + fileBInsert.executeUpdate() + + debug('Inserting data into EATFILEC') + fileCInsert.executeBatch() + + debug('Inserting data into EATFILED') + fileDInsert.executeBatch() + + debug('Inserting data into EATFILEE') + fileEInsert.executeUpdate() + + debug('Inserting data into EATFILEF') + fileFInsert.executeBatch() + + debug('Inserting data into EATFILEG') + fileGInsert.executeBatch() + + #Don't touch the stats table if in debug mode + if not (DEBUG_MODE): + debug('Updating Statistics File') + statsInsert.executeUpdate() + +''' +@param root: An elementtree.Element object referring to the root tag of the XML document + +Most of the files that are being updated on the AS400 require the Sid of the containing XML tag for some reason. +From what I can tell the Sid exists because there are several top level tags that are all named 'Application' this +is not a problem however because they all have a different Sid. Since we know the names of the child tags +that contain the data we are after we can loop through the top level tags and check their children for the appropriate +tag names. When one is found that is used a relevant variable is set to the Sid of the parent tag. + +It seems to me like these will never change and I could have just made them constants. Might as well play it safe right? +''' +def getSids(root): + global transferObjectSid + global applicationBoxMotionSid + global applicationMachineSid + global applicationCalcJCSimuSid + #Get sid values for the TransferObject tag, and the 3 Application tags that contain AddBoxMotion, MachineDefinition, and CalcJCSimu + #ForLoopCeption + for child in root: + if(child.tag == 'Entries'): + for subChild in child: + if(subChild.tag == 'TransferObject'): + transferObjectSid = subChild.get('sid') + elif(subChild.tag == 'Application'): + for subSubChild in subChild: + if(subSubChild.tag == 'AddBoxMotion'): + applicationBoxMotionSid = subChild.get('sid') + elif(subSubChild.tag == 'MachineDefinition'): + applicationMachineSid = subChild.get('sid') + elif(subSubChild.tag == 'CalcJCSimu'): + applicationCalcJCSimuSid = subChild.get('sid') + +''' +@param root: An elementtree.Element object referring to the root tag of the XML document + +This method parses the data required for EATFILEA on the AS400 +This data includes: + Pattern Number + Picks + Last Saved Time + Horizontal Warp Ends + Vertical Repeat + +The data is returned in a list: +data = [patternNumber, Picks, Last_Saved_Time, Horizontal_Warp_Ends, Vertical_Repeat] +''' +def getFileAData(root): + global patternNumber + ret = [] + ret.append(base64.decodestring(root.find(PATTERN_NUMBER_TAG).text)) + ret.append(transferObjectSid) + ret.append(root.find(PICKS_TAG).text) + x = base64.decodestring(root.find(TIME_STAMP_TAG_A).text).split(' ') + ret.append(x[0] + x[1]) + ret.append(root.find(HORIZ_WARP_ENDS_TAG).text) + ret.append(root.find(VERT_REPEAT_TAG).text) + + debug('\t\tPattern Number: ' + ret[0]) + debug('\t\tSid: ' + ret[1]) + debug('\t\tPicks: ' + ret[2]) + debug('\t\tTime Stamp: ' + ret[3]) + debug('\t\tHorizontal Warp Ends: ' + ret[4]) + debug('\t\tVertical Repeat: ' + ret[5]) + + try: + ret.append(base64.decodestring(root.find(NOTE_TAG).text)) #This note is almost never populated, none-the-less we will try.. + except AttributeError: + ret.append(' ') + + debug('\t\tNote: ' + ret[6]) + patternNumber = ret[0] + return ret + +''' +@param root: An elementtree.Element object referring to the root tag of the XML document + +This method parses the data required for EATFILEA on the AS400 +This data includes: + Pattern Number + Sid Number of the containing XML tag + Sequence number + Card Count, these always seem to be the same values as Ground Cards in EATFILEG + +After looking through the programs on the AS400 that interpret the data that is sent; I've found that only +files A, B, E, F, and G are actually used. C and D just seem to be cleared. As a result of this, I've made it +so that if the XML tags for the data are not found in the file it is just ignored instead of halting processing. + +02/18/14 - 6 months later; haven't run into any problems ignoring files C and D if no data is located. + +patternNumber must have been previously determined by getFileAData()! +The appropriate Sid number must have been previously determined by getSids()! + +The data is returned in a 2D list: +data = [[patternNumber, Sid, Sequence, Card_Count], + [patternNumber, Sid, Sequence, Card_Count], + [patternNumber, Sid, Sequence, Card_Count]] +''' +def getFileCData(root): + debug('\tPattern\tSID\tSequence\tCard Count') + ret = [] + boxMotionColorList = root.find(SEQUENCE_NUM_TAG_C) + + try: #This data seems inconsequential to the process, if boxMotionColorList ends up None we'll return an empty list instead of halting processing + for listEntry in boxMotionColorList: + idno = listEntry.get('ID') + cardCount =listEntry.find(CARD_COUNT_TAG).text + row = [patternNumber, applicationBoxMotionSid, idno, cardCount] + ret.append(row) + debug('\t' + row[0] + '\t' + row[1] + '\t' + row[2] + '\t\t' + row[3]) + return ret + except TypeError: + debug('\tWarning: No data was located for EATFILEC. You can probably ignore this.') + return [] + except JavaException, e: + debug('Encountered problems while parsing data for file C. You can probably ignore this.') + debug('Error Message: ' + e.toString()) + return [] + + +''' +@param root: An elementtree.Element object referring to the root tag of the XML document + +This method parses the data required for EATFILED on the AS400 +This data includes: + Pattern Number + Sid Number of the containing XML tag + Sequence Number + Artist + Computer Name + Time Stamp + +After looking through the programs on the AS400 that interpret the data that is sent; I've found that only +files A, B, E, F, and G are actually used. C and D just seem to be cleared. As a result of this, I've made it +so that if the XML tags for the data are not found in the file it is just ignored instead of halting processing. + +02/18/14 - 6 months later; haven't run into any problems ignoring files C and D if no data is located. + +patternNumber must have been previously determined by getFileAData()! +The appropriate Sid number must have been previously determined by getSids()! + +The data is returned in a 2D list: +data = [[patternNumber, Sid, Sequence, Artist, Computer, Time_Stamp], + [patternNumber, Sid, Sequence, Artist, Computer, Time_Stamp], + [patternNumber, Sid, Sequence, Artist, Computer, Time_Stamp]] +''' +def getFileDData(root): + debug('\tPattern\tSID\tSequence\tArtist\t\tComputer\t\t\tTime Stamp') + ret = [] + modifcationEntries = root.find(SEQUENCE_NUM_TAG_D) + + try: #This data seems inconsequential to the process, if modifcationEntries ends up None we'll return an empty list instead of halting processing + for listEntry in modifcationEntries: + idno = listEntry.get('ID') + artist = base64.decodestring(listEntry.find(ARTIST_TAG).text) + computer = base64.decodestring(listEntry.find(COMPUTER_NAME_TAG).text) + timestamp = base64.decodestring(listEntry.find(TIME_STAMP_TAG_D).text).split(' ') + timestamp = timestamp[0] + timestamp[1] + row = [patternNumber, transferObjectSid, idno, artist, computer, timestamp] + ret.append(row) + debug('\t' + row[0] + '\t' + row[1] + '\t' + row[2] + '\t\t' + row[3] + '\t' + row[4] + '\t\t\t' + row[5]) + except TypeError: + debug('\tWarning: No data was located for EATFILED. You can probably ignore this.') + except JavaException: + debug('\tWarning: Modification history corrupt. You can probably ignore this.') + return ret + +''' +@param root: An elementtree.Element object referring to the root tag of the XML document + +This method parses the data required for EATFILEF on the AS400 +This data includes: + Pattern Number + Sid Number of the containing XML tag + Yarn Sequence Number + Yarn Type and Yarn Color in one field + +patternNumber must have been previously determined by getFileAData()! +The appropriate Sid number must have been previously determined by getSids()! + +The data is returned in a 2D list: +data = [[patternNumber, Sid, Sequence, Yarn], + [patternNumber, Sid, Sequence, Yarn], + [patternNumber, Sid, Sequence, Yarn]] +''' +def getFileFData(root): + debug('\tPattern\tSid\tSequence\tYarn') + ret = [] + weftYarnList = root.find(SEQUENCE_NUM_TAG_F) + for listEntry in weftYarnList: + idno = listEntry.get('ID') + yarn = base64.decodestring(listEntry.find(PAT_NUM_TAG_F).text) + row = [patternNumber, applicationCalcJCSimuSid, idno, yarn] + ret.append(row) + debug('\t' + row[0] + '\t' + row[1] + '\t' + row[2] + '\t\t' + row[3]) + return ret + +''' +@bug: Due to the inadequate size of the card fields (3 digits) we are unable to process patterns that have + more than 999 cards for a single position. + +@param root: An elementtree.Element object referring to the root tag of the XML document + +This method parses the data required for EATFILEG on the AS400 +This data includes: + Pattern Number + Sid Number of the containing XML tag + Yarn Sequence Number + Ground Cards + Tissue Cards + +patternNumber must have been previously determined by getFileAData()! +The appropriate Sid number must have been previously determined by getSids()! + +The data is returned in a 2D list: +data = [[patternNumber, Sid, Sequence, Ground_Cards, Tissue_Cards], + [patternNumber, Sid, Sequence, Ground_Cards, Tissue_Cards], + [patternNumber, Sid, Sequence, Ground_Cards, Tissue_Cards]] +''' +def getFileGData(root): + debug('\tPattern\tSid\tSequence\tGround Cards\tTissue Cards') + ret = [] + yarnNums = root.findall(YARN_NUM_TAG) + tissueCards = root.findall(TISSUE_CARD_TAG) + groundCards = root.findall(GROUND_CARD_TAG) + + for i in range(0,len(yarnNums)): + row = [patternNumber, applicationBoxMotionSid, yarnNums[i].text, groundCards[i].text, tissueCards[i].text] + ret.append(row) + debug('\t' + row[0] + '\t' + row[1] + '\t' + row[2] + '\t\t' + row[3] + '\t\t' + row[4]) + if(int(row[3]) > 999) or (int(row[4]) > 999): + raise RuntimeError('ERROR: Card counts are too large for the fields on the AS400 (> 999). You\'ll have to key this one in by hand. Sorry!') + return ret + +''' +XSD files contain schema information for XML files. They can be used to validate the structure of +an associated XML file. We do not use them for anything so this method is called during startup to +clear them out of the folder. They are generated by the EAT software when the designers export a +pattern in XML format. +''' +def deleteXSDFiles(): + for f in glob.glob(NETWORK_PATH + "\*.xsd"): #Generate a list of all XSD files in the network folder + os.remove(f) #Delete each XSD file + +''' +Delete log files after KEEP_LOG_FILES days. Called on startup +''' +def pruneLogFiles(): + filelist = glob.glob(NETWORK_PATH + "\\logs\\" + LOG_FILE_REGEX) #Generate a list of all text files in the log folder + then = (datetime.datetime.now() - timedelta(days=KEEP_LOG_FILES)) #logs older than this will be deleted + for f in filelist: + try: + #Super dirty hack, don't care. The worst that can happen is that some logs won't get deleted + ymd = f.split('\\')[LOG_FOLDER_DEPTH].split('.')[0].split('-') + now = datetime.datetime(int(ymd[2]), int(ymd[0]), int(ymd[1])) + if(now < then): + os.remove(f) + except ValueError: + pass + +''' +@param qtm: + +Builds a list of files to be parsed and uploaded to the AS400 + +Loops through every row in the table, if the process boolean is set the full path is +prepended to the file name and added to the list that is returned by the method. +''' +def getFilesSelectedForProcessing(qtm): + ret = [] + for row in qtm.cache: + if(row[1]): + ret.append(NETWORK_PATH + row[0]) + return ret + +''' +This method is the actionPerformed property for the 'Process!' button. Everything not having to do +with the UI happens starting from here. + +1. The log file is opened and a dashed separator line is written to help distinguish sessions in the log file +2. getFilesSelectedForProcessing() is called to create a list of files that were selected in the table by the user +3. prepareToSend() is called which opens a connection with the AS400 and pre-compiles all of the SQL statements +4. parseAndSend() is called for each file in the list. This is where most of the processing occurs. +5. QCMDEXC is invoked via a stored procedure call in order to execute STMLIB/XMLPARSE. This causes the data uploaded to be processed by the AS400 right away. +6. The open connection with the AS400 is closed +7. The log file buffer is flushed in the finally block, to make sure any error information gets written to the log file. +''' +def process(event): + global logFile + global currentlyProcessing + try: + frame.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)) #Change mouse icon to loading icon for duration of this method + + logFile = openLogFile() + logFile.write('<--------------------------EAT DATA UPLOAD REQUESTED-------------------------->\n') + debug('Sunbury Textile Mills - XMLMapper v' + VERSION_ID + ' - Author: Wesley Ray') + + files = getFilesSelectedForProcessing(qtm) + debug(getpass.getuser() + ' selected ' + str(len(files)) + ' file(s) for processing') + for f in files: + debug('\t\t' + f) + + prepareToSend() #Open connection with server, pre-compile SQL statements + + for inFile in files: + currentlyProcessing = inFile.split('\\')[4] + parseAndSend(inFile) #Parse XML data and upload to server for each file + qtm.setValueAt(False, qtm.cache.index([currentlyProcessing, True]), 1) #Un-check successfully processed files + debug(' ') + + debug('Processing completed normally! AS400 connection closed.') + JOptionPane.showMessageDialog(None, 'Processing of ' + str(len(files)) + ' file(s) completed with no errors reported!', 'Success!', JOptionPane.INFORMATION_MESSAGE) + + except Exception, e: + frame.setCursor(Cursor.getDefaultCursor()) + debug('An error has occurred while processing ' + currentlyProcessing) + debug('Only the files processed successfully up to this point have been sent to the AS400!') + debugError(e) + JOptionPane.showMessageDialog(frame, 'An error occurred while processing ' + currentlyProcessing + '\n\rProbably a bogus XML document from the designer, have them re-send it.\n\rIf the problem persists call Wes @ Ext. 234', "An Error Has Occurred!", + JOptionPane.ERROR_MESSAGE) + except JavaException, e: + frame.setCursor(Cursor.getDefaultCursor()) + debug('An error has occurred while processing ' + currentlyProcessing) + debug('Only the files processed successfully up to this point have been sent to the AS400!') + debug('Error Message: ' + e.toString()) + JOptionPane.showMessageDialog(frame, 'An error occurred while processing ' + currentlyProcessing + '\n\rProbably a bogus XML document from the designer, have them re-send it.\n\rIf the problem persists call Wes @ Ext. 234', "An Error Has Occurred!", + JOptionPane.ERROR_MESSAGE) + finally: + fileAInsert.close() #Close PreparedStatement Objects + fileBInsert.close() + fileCInsert.close() + fileDInsert.close() + fileEInsert.close() + fileFInsert.close() + fileGInsert.close() + + try: + blah = connection.createStatement().executeQuery('select count(*) from wes.xmlstats') + blah.next() + blah = blah.getObject(1) + debug(str(blah) + ' files processed since 05/15/14!') + debug('Attempting to force data update on AS400...') + forceUpdate.execute() + debug('Force update successful! The printouts should be on P7 now.') + except JavaException, e: + debug('Force Update Error: ' + e.toString()) + JOptionPane.showMessageDialog(frame, 'Could not force update!\n\rThe data should be picked up automatically in about 5 minutes.', JOptionPane.ERROR_MESSAGE) + + forceUpdate.close() #Close CallableStatement object + connection.close() #Close Connection Object + + logFile.flush() + qtm.fireTableChanged(None) #Repaint Table + frame.setCursor(Cursor.getDefaultCursor()) + +''' +This method is the actionPerformed property for both the 'Select' and 'Deselect' buttons. + +Basically loops through each row in the table data and sets the process boolean equal to +what (event.getSource() is selectButton) evaluates to; which will be True if select all +was clicked and false if deselect all was clicked. +''' +def selectAll(event): + for row in qtm.cache: + row[1] = (event.getSource() is selectButton) #Set 'process?' boolean for each row in the table + qtm.fireTableChanged(None) #Repaint Table + +''' +I found out that Kathy will typically not delete the XML files directly after sending them, rather, she keeps them +until the design process is finished so that if there is an issue she doesn't need to have the designers re-send +the XML. So I got rid of the delete prompt in the process method and added a button to 'Delete Selected' files. + +All this does is get the selected file list from the file table and delete them from the network folder. +''' +def deleteSelected(event): + global logFile + logFile = openLogFile() + dialogResult = JOptionPane.showConfirmDialog(frame, 'Are you sure you would like to delete the selected files? This _CANNOT_ be undone!', 'Delete Files', JOptionPane.YES_NO_OPTION) + if(dialogResult == JOptionPane.YES_OPTION): + for f in getFilesSelectedForProcessing(qtm): #Fetch list of selected files from the file table + os.remove(f) #Delete them + debug(f + ' Deleted!') #Log this + qtm.loadTable(glob.glob(NETWORK_PATH + "\*.xml")) #Refresh file table + logFile.flush() + logFile.close() + +def refreshList(event): + qtm.loadTable(glob.glob(NETWORK_PATH + "\*.xml")) #Call the loadTable method with a list of XML files in the network folder + +if __name__ == '__main__': + for line in ASCII_ART: + print line + deleteXSDFiles() #Delete all XSD files in the network folder + pruneLogFiles() #Delete log files older than 7 days + + frame = JFrame('Sunbury Textile Mills - XML Mapper') #Create a new JFrame() + frame.setIconImage(ImageIcon('icon.png').getImage()) #Found an XML icon on the interwebs, better than the java logo anyways... + UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName()) #Set look-and-feel to Windows-ish + frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE) #Call System.exit on frame closing + frame.setSize(340, 340) #Good dimensions as determined by me + frame.setLayout(BorderLayout()) #Pretty simple GUI, we'll use BorderLayout() + + table = JTable() #Create file table + table.setModel(FileTableModel()) #Load my AbstractTableModel implementation I created for this program + scrollPane = JScrollPane(table) #Create a JScrollPane to put the table in + table.setFillsViewportHeight(True) #Make table always fill the scrollPane + table.setRowSelectionAllowed(False) #Disallow row selection, we are using in-cell JCheckBoxes instead + qtm = table.getModel() #Get a reference to the TabelModel so we can access the table data and loadTable() method + qtm.loadTable(glob.glob(NETWORK_PATH + "\*.xml")) #Call the loadTable method with a list of XML files in the network folder + + processButton = JButton('Process!', actionPerformed=process) #JButton to initiate EAT data upload + selectButton = JButton('Select All', actionPerformed=selectAll) #JButton to select all files in the list + deselectButton = JButton('Deselect All', actionPerformed=selectAll) #JButton to deselect all selected files in the list + refreshButton = JButton('', actionPerformed=refreshList) + refreshButton.setIcon(ImageIcon(ImageIO.read(File('refresh.png')))) + refreshButton.setToolTipText('Refresh file list. Useful for when designers save files to the folder after you start this program.') + deleteButton = JButton('', actionPerformed=deleteSelected) + deleteButton.setIcon(ImageIcon(ImageIO.read(File('delete.png')))) + deleteButton.setToolTipText('Delete selected files') + + topPanel = JPanel() #Since we are using BorderLayout, create a JPanel to contain all of the JButtons + topPanel.add(selectButton) + topPanel.add(deselectButton) + topPanel.add(refreshButton) + topPanel.add(deleteButton) + + frame.add(topPanel, BorderLayout.NORTH) #Add the button JPanel to BorderLayout.NORTH + frame.add(scrollPane, BorderLayout.CENTER) #Add the JScrollPane to BorderLayout.CENTER + frame.add(processButton, BorderLayout.SOUTH) #Add the 'Process!' JButton to BorderLayout.SOUTH + #frame.pack() #JTable's behave strangely with this it seems, I figured out good dimensions myself + frame.setLocationRelativeTo(None) #Center the frame on screen + frame.setResizable(False) #Do not allow resizing the frame + frame.setVisible(True) #Display the frame diff --git a/EATXMLmapper/STM/TabelModel.py b/EATXMLmapper/STM/TabelModel.py new file mode 100644 index 0000000..c86e76b --- /dev/null +++ b/EATXMLmapper/STM/TabelModel.py @@ -0,0 +1,64 @@ +''' +Created on Sep 11, 2013 +@author: Wesley Ray +''' + +from java.lang import Boolean, String +from javax.swing.table import AbstractTableModel + +class FileTableModel(AbstractTableModel): + ''' + Implementation of AbstractTableModel for displaying a list of file names with a check box for each of them. + ''' + headers = ['File', 'Process?'] + cache = [] + + #@Override + def getColumnName(self, i): + return self.headers[i] + + #@Override + def getRowCount(self): + if (self.cache == None): + return 0 + return len(self.cache) + + #@Override + def getColumnCount(self): + return len(self.headers) #Never changes, could just be: return 2 + + #@Override + def getValueAt(self, rowIndex, columnIndex): + row = self.cache[rowIndex] + return row[columnIndex] + + #@Override + def isCellEditable(self, rowIndex, columnIndex): + #We have to do this to allow the state of the check boxes to be changed by the user + if(columnIndex == 1): + return True + else: + return False + + #@Override + def setValueAt(self, aValue, rowIndex, columnIndex): + #This toggles the state of the check box in the given rowIndex + if(columnIndex == 1): + self.cache[rowIndex][columnIndex] = not(self.cache[rowIndex][columnIndex]) + + #@Override + def getColumnClass(self, column): + if(column == 1): + return Boolean #This causes the second column to be rendered as check boxes + else: + return String + + def loadTable(self, files): + self.cache = [] + for x in files: + s = x.split('\\') + row = [s[4], Boolean.FALSE] + self.cache.append(row) + + self.fireTableChanged(None) + self.fireTableStructureChanged() \ No newline at end of file