Adding multiple INDEX fields

Author: ibmmqmet

CHANGES.md

@@ -59,3 +59,8 @@ the documentation and product header files claim.
 * Some SMF integers are too large for the INTEGER datatype. The unsigned
 32-bits compared to the SQL which is signed 32-bit, So switching
 to always use BIGINT.
+
+Mar 2018
+* Add code to help add INDEX to columns in the SQL DDL output
+* Add optimisation to Makefiles
+* Even 64-bit values may be too large when they are not really counters

bin/aix/convH

@@ -4,7 +4,7 @@ plat=`uname`
 
 if [ "$plat" = "AIX" ]
 then
-  flags="-q32"
+  flags="-q32 -qmaxmem=-1"
   cc="xlc"
   targdir="../bin/aix"
 else
@@ -15,6 +15,8 @@ fi
 if [ "$CCDEBUG" != "" ]
 then
   flags="$flags -g"
+else
+  flags="$flags -O"
 fi
 
 export PLATFLAGS=$flags CC=$cc VERS=904

bin/aix/mqsmfcsv

@@ -1,51 +1,51 @@
-CFLAGS=-nologo /D_CRT_SECURE_NO_WARNINGS /Zp1 /J
-VERS=904
-SRC = mqsmf.c \
-      smfDDL.c \
-      smfDate.c \
-      smfConv.c \
-      printDEBUG.c \
-      printQ5ST.c    \
-      printQCST.c    \
-      printQCCT.c  \
-      printQCTADP.c\
-      printQCTDNS.c\
-      printQCTDSP.c\
-      printQCTSSL.c\
-      printQESD.c  \
-      printQEST.c  \
-      printQIS1.c  \
-      printQIST.c  \
-      printQJST.c  \
-      printQLST.c  \
-      printQMAC.c  \
-      printQMST.c  \
-      printQPST.c  \
-      printQSGM.c  \
-      printQSPH.c  \
-      printQSRS.c  \
-      printQSST.c  \
-      printQTST.c  \
-      printWQ.c    \
-      printWTAS.c  \
-      printWTID.c
-
-HDR = mqsmfstruc.h \
-      mqsmf.h
-
-mqsmfcsv.exe: $(SRC) $(HDR) Makefile.win
-	rm -f $@
-	$(CC) $(SRC) -I. /Fe$@ $(CFLAGS) -DCSQDSMF_VERSION=$(VERS)
-	del /q *.obj
-	copy $@ ..\bin\win
-
-mqsmfstruc.h: csqdsmfc-$(VERS).h convH.exe dummy
-	convH < csqdsmfc-$(VERS).h > $@
-
-convH.exe: convH.c
-	$(CC) /Fe$@ convH.c
-	copy $@ ..\bin\win
-
-
-dummy:
-
+CFLAGS=-nologo /D_CRT_SECURE_NO_WARNINGS /Zp1 /J /O2
+VERS=904
+SRC = mqsmf.c \
+      smfDDL.c \
+      smfDate.c \
+      smfConv.c \
+      printDEBUG.c \
+      printQ5ST.c    \
+      printQCST.c    \
+      printQCCT.c  \
+      printQCTADP.c\
+      printQCTDNS.c\
+      printQCTDSP.c\
+      printQCTSSL.c\
+      printQESD.c  \
+      printQEST.c  \
+      printQIS1.c  \
+      printQIST.c  \
+      printQJST.c  \
+      printQLST.c  \
+      printQMAC.c  \
+      printQMST.c  \
+      printQPST.c  \
+      printQSGM.c  \
+      printQSPH.c  \
+      printQSRS.c  \
+      printQSST.c  \
+      printQTST.c  \
+      printWQ.c    \
+      printWTAS.c  \
+      printWTID.c
+
+HDR = mqsmfstruc.h \
+      mqsmf.h
+
+mqsmfcsv.exe: $(SRC) $(HDR) Makefile.win
+	rm -f $@
+	$(CC) $(SRC) -I. /Fe$@ $(CFLAGS) -DCSQDSMF_VERSION=$(VERS)
+	del /q *.obj
+	copy $@ ..\bin\win
+
+mqsmfstruc.h: csqdsmfc-$(VERS).h convH.exe dummy
+	convH < csqdsmfc-$(VERS).h > $@
+
+convH.exe: convH.c
+	$(CC) /Fe$@ convH.c
+	copy $@ ..\bin\win
+
+
+dummy:
+

bin/linux/mqsmfcsv

@@ -154,7 +154,7 @@ extern void printWTID  (wtid *, unsigned char *);
 extern void openDDL    (char *);
 extern void closeDDL   (char *);
 extern void printDDL   (char *,int, int);
-extern void setIndex   (char *);
+extern void addIndex   (char *);
 
 
 /*******************************************************/
@@ -221,6 +221,16 @@ extern char *strCfStatType  (int v);
 #define DDL_SUS      (4)
 #define DDL_DATETIME (5)
 
+/************************************************************************/
+/* Some "unsigned 64-bit" in the SMF are really pointers or correlators */
+/* No real counter is likely to reach the top bit. But treating them as */
+/* unsigned values in these cases is causing overflow when loading into */
+/* DB2 because SQL 64-bit quantities are always signed with no option   */
+/* for the C datatype. So as an expediency, we will drop the sign bit   */
+/* which should not affect any real calculations.                       */
+/************************************************************************/
+#define DROPSIGN 0x7FFFFFFFFFFFFFFF
+
 /**********************************************************/
 /* Macros to build the printable lines for CSV output.    */
 /* Separate macros for each data type and when we need    */
@@ -265,7 +275,7 @@ extern char *strCfStatType  (int v);
 
 #define ADDU64(h,v) \
   ADDHEAD(h,DDL_I64,0); \
-  ADDDATA("%llu,",conv64(v))
+  ADDDATA("%lld,",(DROPSIGN & conv64(v)))
 
 #define ADDS64IDX(h,idx,v) \
   if (first) sprintf(tmpHead,"%s {%s}",h,idx); \
@@ -275,7 +285,7 @@ extern char *strCfStatType  (int v);
 #define ADDU64IDX(h,idx,v) \
   if (first) sprintf(tmpHead,"%s {%s}",h,idx); \
   ADDHEAD(tmpHead,DDL_I64,0); \
-  ADDDATA("%llu,",conv64(v))
+  ADDDATA("%lld,",(DROPSIGN & conv64(v)))
 
 #define ADDS32(h,v) \
   ADDHEAD(h,DDL_I,0); \
@@ -395,6 +405,11 @@ extern char *strCfStatType  (int v);
   } \
   COMMON_BLOCK
 
+#define ADDINDEX(n) \
+  if (first) { \
+    addIndex(n); \
+  }
+
 #define SMFPRINTSTOP \
   if (first && newFile && printHeaders)             \
   {                      \

bin/win/mqsmfcsv.exe

@@ -38,7 +38,7 @@ void printWQ(wq *p)
   ADDSTRB ("Correlation"    ,p->correl, 16);
   ADDSTREN("Open_Name",p->objname,48);
   ADDSTREN("Base_Name",p->basename,48);
-  setIndex("Base_Name");
+  ADDINDEX("Base_Name");
   ADDTIME ("Open"     ,p->opentime);
   ADDTIME ("Close"    ,p->clostime);
 

src/M

@@ -17,48 +17,79 @@
 
 static FILE *fp = NULL;
 char *comma = "";
+char *schema = "MQSMF";
 
-char *primaryKey = NULL;
+#define INDEXSETSIZE 128
+char *indexSet[INDEXSETSIZE] = {0};  
+int   nextIndex = 0;
 
 #define COL_HEAD_LEN (64)
 
 static char *format(char *);
 
+/*
+ * The DDL generated here is very simple. The
+ * table is created with a fixed name and relevant
+ * datatypes. If any index columns have been named
+ * then those are added to the DDL with each table.
+ * There is no "Primary key" defined.
+ */
+
 void openDDL(char *name)
 {
    BOOL newFile;
+   int i;
+
+   for (i=0;i<INDEXSETSIZE;i++)
+   {
+     indexSet[i] = NULL;
+   }
+   nextIndex = 0;
 
-   primaryKey = NULL;
    if (!fp)
    {
      fp = fopenext("MQTABLES","ddl", &newFile);
      if (!fp) /* fopenext will print detailed error message */
        exit(1);
    }
-   fprintf(fp,"DROP   TABLE MQSMF.%s;  \n",name);
+   fprintf(fp,"DROP   TABLE %s.%s;  \n",schema,name);
 
    /* If you want more sophisticated SQL, then you could change this */
-   fprintf(fp,"CREATE TABLE MQSMF.%s (\n",name);
+   fprintf(fp,"CREATE TABLE %s.%s (\n",schema,name);
    comma = " ";                               /* Start with it blank */
 }
 
 void closeDDL(char *name)
 {
+  int i;
   if (fp)
   {
+    /* Could add "... IN <tablespace>" to this if needed */
     fprintf(fp,");\n");
-    if (primaryKey) { 
-      fprintf(fp,"CREATE INDEX %s ON MQSMF.%s(%s);\n",primaryKey,name,primaryKey);
+    for (i=0;i<INDEXSETSIZE;i++)
+    {
+      char *c = indexSet[i];
+      if (c) { 
+        fprintf(fp,"CREATE INDEX %s ON %s.%s(%s);\n",c,schema,name,c);
+      }
     }
     fprintf(fp,"\n");
     fflush(fp);
   }
   return;
 }
 
-void setIndex(char *key) 
+void addIndex(char *key) 
 {
-  primaryKey = key;
+  if (nextIndex <INDEXSETSIZE) {
+    indexSet[nextIndex] = key;
+    nextIndex++;
+  }
+  else 
+  {
+    /* Extremely unlikely that we'll have this number of index columns for a table */
+    printf("Warning: Cannot add %s to table index. Need to increase INDEXSETSIZE\n",key);
+  }
   return;
 }
 
@@ -94,11 +125,10 @@ void printDDL(char *name, int type, int len)
       fprintf(fp,"%s %s \t CHAR(%d)\n",comma,format(nameCopy),len);
     }
     break;
-  case DDL_SUS:
+  case DDL_SUS: /* a seconds/microseconds field */
     p2 = strstr(nameCopy,"(US)");
     if (p2) *p2 = 0;
     fprintf(fp,"%s %s_us \t BIGINT\n",comma,format(nameCopy));
-
     break;
 
   case DDL_DATETIME:
@@ -181,7 +211,7 @@ static char *format(char *name)
 
   /* Check for column name length - I've seen this as a limit in some DBs */
   /* Leave room for _DATE etc as prefix */
-  if(strlen(nameNoDup) > 27)
+  if (strlen(nameNoDup) > 27)
     printf("Warning: Column %s may be too long\n",nameNoDup);
 
   return nameNoDup;