JSON SerDe: date column in any time zone - for at least Hive-3.1+ (#183)

Author: randallwhitman

hive/src/main/java/com/esri/hadoop/hive/serde/BaseJsonSerDe.java

@@ -202,6 +202,9 @@ public Object deserialize(Writable json_in) throws SerDeException {
 } catch (IOException e) {
 // TODO Auto-generated catch block
 e.printStackTrace();
+ } catch (java.text.ParseException jtpe) {
+ // probably should set field to null?
+ jtpe.printStackTrace();
 }
 
 return row;
@@ -321,7 +324,7 @@ private void generateJsonFromWritable(Writable value, int fieldIndex, String lab
  PrimitiveObjectInspector poi, JsonGenerator jsonGen)
 throws JsonProcessingException, IOException {
 Object prim = poi.getPrimitiveJavaObject(value);
-Long epoch = HiveShims.getPrimitiveEpoch(prim, tz);
+Long epoch = HiveShims.getPrimitiveEpoch(prim);
 if (epoch == null) { // anything but a recognized DATE or TIMESTAMP
 jsonGen.writeObjectField(label, prim);
 } else {
@@ -335,43 +338,45 @@ private void generateJsonFromWritable(Writable value, int fieldIndex, String lab
  // Parse OGCGeometry from JSON
 abstract protected OGCGeometry parseGeom(JsonParser parser);
 
-private java.sql.Date parseDate(JsonParser parser) throws JsonParseException, IOException {
-java.sql.Date jsd = null;
+// See HIVE-12192.
+ // Complete handling of absent/invalid date as SQL NULL would be broader scope.
+private long parseDate(JsonParser parser)
+ throws JsonParseException, IOException, java.text.ParseException {
 if (JsonToken.VALUE_NUMBER_INT.equals(parser.getCurrentToken())) {
-// DateWritable#daysToMillis adjusts the numerical/epoch time
-// to midnight in the local time zone.	See HIVE-12192.
-// Attempt to compensate, when date provided as epoch, which is unambiguously UTC.
 long epoch = parser.getLongValue();
-jsd = new java.sql.Date(epoch - tz.getOffset(epoch));
-} else try {
+ return epoch;
+} else {
 long epoch = parseTime(parser.getText(), "yyyy-MM-dd");
-jsd = new java.sql.Date(epoch + 43200000); // midday rather than midnight
-} catch (java.text.ParseException e) {
-// null
-}
-return jsd;
+return epoch + 43200000; // midday rather than midnight
+ }
 }
 
-private java.sql.Timestamp parseTime(JsonParser parser) throws JsonParseException, IOException {
-java.sql.Timestamp jst = null;
+ // Complete handling of absent/invalid date-time as SQL NULL would be broader scope.
+private long parseTime(JsonParser parser) throws JsonParseException, IOException, java.text.ParseException {
+ long epoch = -9999L;
+ java.text.ParseException jtpe = null;
 if (JsonToken.VALUE_NUMBER_INT.equals(parser.getCurrentToken())) {
-long epoch = parser.getLongValue();
-jst = new java.sql.Timestamp(epoch);
+epoch = parser.getLongValue();
 } else {
 String value = parser.getText();
 int point = value.indexOf('.');
 String dateStr = (point < 0) ? value : value.substring(0,point+4);
 String[] formats = {"yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm", "yyyy-MM-dd"};
 for (String format: formats) {
 try {
-jst = new java.sql.Timestamp(parseTime(dateStr, format));
+epoch = parseTime(dateStr, format);
+ jtpe = null;
 break;
-} catch (java.text.ParseException e) {
-// remain null after this attempted format
+} catch (java.text.ParseException exc) {
+ if (null == jtpe)
+ jtpe = exc;
 }
 }
 } // else String value
-return jst;
+ if (null == jtpe)
+ return epoch;
+ else
+ throw jtpe;
 }
 
 private long parseTime(String value, String format) throws java.text.ParseException { // epoch
@@ -388,8 +393,10 @@ private long parseTime(String value, String format) throws java.text.ParseExcept
  * @param parser JsonParser pointing to the attribute
  * @throws JsonParseException
  * @throws IOException
+ * @throws ParseException
  */
-private void setRowFieldFromParser(int fieldIndex, JsonParser parser) throws JsonParseException, IOException{
+private void setRowFieldFromParser(int fieldIndex, JsonParser parser) throws
+ JsonParseException, IOException, java.text.ParseException {
 
 PrimitiveObjectInspector poi = (PrimitiveObjectInspector)this.columnOIs.get(fieldIndex);
 if (JsonToken.VALUE_NULL == parser.getCurrentToken())
@@ -420,7 +427,7 @@ private void setRowFieldFromParser(int fieldIndex, JsonParser parser) throws Jso
 case BOOLEAN:
 ((BooleanWritable)row.get(fieldIndex)).set(parser.getBooleanValue());
 break;
-case DATE: // DateWritable stores days not milliseconds.
+case DATE: // DateWritable stores days not milliseconds. See also HIVE-12192.
 HiveShims.setDateWritable(row.get(fieldIndex), parseDate(parser));
 break;
 case TIMESTAMP:

hive/src/main/java/com/esri/hadoop/shims/HiveShims.java

@@ -3,7 +3,9 @@
 import java.lang.reflect.Method;
 import java.util.TimeZone;
 
-
+/**
+ * These shims are internal to Spatial-Framework-for-Hadoop and subject to change without notice.
+ */
 public class HiveShims {
 
 /**
@@ -48,7 +50,7 @@ static String getAsStringOrNull(Class<?> clazz, String constant) {
  /**
  * Classes o.a.h.h.common.type Date & Timestamp were introduced in Hive-3.1 version.
  */
- public static Long getPrimitiveEpoch(Object prim, TimeZone tz) {
+ public static Long getPrimitiveEpoch(Object prim) {
  if (prim instanceof java.sql.Timestamp) {
 return ((java.sql.Timestamp)prim).getTime();
  } else if (prim instanceof java.util.Date) {
@@ -78,9 +80,8 @@ public static Long getPrimitiveEpoch(Object prim, TimeZone tz) {
  * Type DATE was introduced in Hive-0.12 - class DateWritable in API.
  * Class DateWritableV2 is used instead as of Hive-3.1 version.
  */
- public static void setDateWritable(Object dwHive, long epoch
- , TimeZone tz
- ) {
+// See HIVE-12192.
+ public static void setDateWritable(Object dwHive, long epoch) {
  try { // Hive 3.1 and above
  Class<?> dtClazz = Class.forName("org.apache.hadoop.hive.common.type.Date");
  Class<?> dwClazz = Class.forName("org.apache.hadoop.hive.serde2.io.DateWritableV2");
@@ -97,32 +98,7 @@ public static void setDateWritable(Object dwHive, long epoch
  } catch (Exception e2) { // Hive 0.11 and below
  // column type DATE not supported
  throw new UnsupportedOperationException("DATE type");
- }
- }
- } // setDateWritable
-
-/**
- * Type DATE was introduced in Hive-0.12 - class DateWritable in API.
- * Class DateWritableV2 is used instead as of Hive-3.1 version.
- */
- public static void setDateWritable(Object dwHive, java.sql.Date jsd) {
- try { // Hive 3.1 and above
- Class<?> dtClazz = Class.forName("org.apache.hadoop.hive.common.type.Date");
- Class<?> dwClazz = Class.forName("org.apache.hadoop.hive.serde2.io.DateWritableV2");
- Method dtSetImpl = dtClazz.getMethod("setTimeInMillis", long.class);
- Method dwSetImpl = dwClazz.getMethod("set", dtClazz);
- Object dtObj = dtClazz.getConstructor().newInstance();
- dtSetImpl.invoke(dtObj, jsd.getTime());
- dwSetImpl.invoke(dwHive, dtObj);
- } catch (Exception e1) {
- try { // Hive 0.12 and above
- Class<?> dwClazz = Class.forName("org.apache.hadoop.hive.serde2.io.DateWritable");
- Method dwSetImpl = dwClazz.getMethod("set", java.sql.Date.class);
- dwSetImpl.invoke(dwHive, jsd);
- } catch (Exception e2) { // Hive 0.11 and below
- // column type DATE not supported
- throw new UnsupportedOperationException("DATE type");
- }
+ }
  }
  } // setDateWritable
 
@@ -151,29 +127,4 @@ public static void setTimeWritable(Object twHive, long epoch) {
  }
  } // setTimeWritable
 
-/**
- * Type TIMESTAMP was introduced in Hive-0.12 - class TimestampWritable in API.
- * Class TimestampWritableV2 is used instead as of Hive-3.1 version.
- */
- public static void setTimeWritable(Object twHive, java.sql.Timestamp jst) {
- long epoch = jst.getTime();
- try { // Hive 3.1 and above
- Class<?> ttClazz = Class.forName("org.apache.hadoop.hive.common.type.Timestamp");
- Class<?> twClazz = Class.forName("org.apache.hadoop.hive.serde2.io.TimestampWritableV2");
- Method ttSetImpl = ttClazz.getMethod("setTimeInMillis", long.class);
- Method twSetImpl = twClazz.getMethod("set", ttClazz);
- Object ttObj = ttClazz.getConstructor().newInstance();
- ttSetImpl.invoke(ttObj, epoch);
- twSetImpl.invoke(twHive, ttObj);
- } catch (Exception e1) {
- try { // Hive 0.12 and above
- Class<?> twClazz = Class.forName("org.apache.hadoop.hive.serde2.io.TimestampWritable");
- Method twSetImpl = twClazz.getMethod("set", java.sql.Timestamp.class);
- twSetImpl.invoke(twHive, new java.sql.Timestamp(epoch));
- } catch (Exception e2) { // Hive 0.11 and below
- // column type TIMESTAMP not supported
- throw new UnsupportedOperationException("TIMESTAMP type");
- }
- }
- } // setTimeWritable
 }

hive/src/test/java/com/esri/hadoop/hive/serde/TestEsriJsonSerDe.java

@@ -3,7 +3,6 @@
 import org.junit.Assert;
 import org.junit.Test;
 
-import java.lang.reflect.Method;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Properties;
@@ -65,26 +64,37 @@ public void TestIntWrite() throws Exception {
 
 @Test
 public void TestEpochWrite() throws Exception {
-ArrayList<Object> stuff = new ArrayList<Object>();
 Properties proptab = new Properties();
 proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
 proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
 AbstractSerDe jserde = mkSerDe(proptab);
 StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
+long day = 24*3600*1000; // DateWritable represents days not milliseconds.
 
-// {"attributes":{"when":147147147147}}
-long epoch = 147147147147L;
- long zoned = epoch - TimeZone.getDefault().getOffset(epoch);
- java.sql.Date expected = new java.sql.Date(zoned);
- addWritable(stuff, expected);
+long epoch = 1641535200000L; // 2022-01-07 06:00 UTC
+ //long zoned = epoch - TimeZone.getDefault().getOffset(epoch);
+ java.sql.Date jsd = new java.sql.Date(epoch); // zoned?
+ArrayList<Object> stuff = new ArrayList<Object>();
+ addWritable(stuff, jsd);
  Writable jsw = jserde.serialize(stuff, rowOI);
 JsonNode jn = new ObjectMapper().readTree(((Text)jsw).toString());
 jn = jn.findValue("attributes");
 jn = jn.findValue("when");
-java.sql.Date actual = new java.sql.Date(jn.getLongValue());
-long day = 24*3600*1000; // DateWritable represents days not milliseconds.
+Assert.assertEquals(epoch/day, jn.getLongValue()/day);
+
+epoch = 1641578400000L; // 2022-01-07 18:00 UTC
+ //long zoned = epoch - TimeZone.getDefault().getOffset(epoch);
+ jsd = new java.sql.Date(epoch); // zoned?
+stuff = new ArrayList<Object>();
+ addWritable(stuff, jsd);
+ jsw = jserde.serialize(stuff, rowOI);
+jn = new ObjectMapper().readTree(((Text)jsw).toString());
+jn = jn.findValue("attributes");
+jn = jn.findValue("when");
+ System.err.println(jn);
 Assert.assertEquals(epoch/day, jn.getLongValue()/day);
 }
+
 @Test
 public void TestTimeWrite() throws Exception {
  ArrayList<Object> stuff = new ArrayList<Object>();
@@ -171,7 +181,6 @@ public void TestDateParse() throws Exception {
 Object row = jserde.deserialize(value);
 StructField f0 = rowOI.getStructFieldRef("when");
 Object fieldData = rowOI.getStructFieldData(row, f0);
-
  Assert.assertEquals(dateStr, iso8601FromWritable(fieldData));
  dateStr = "2017-05-05";
  value.set("{\"attributes\":{\"when\":\"" + dateStr + "\"}}");
@@ -191,18 +200,20 @@ public void TestEpochParse() throws Exception {
 proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
 jserde.initialize(config, proptab);
  StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
+ // Half a day apart to test both a.m. & p.m. whether in East or West
 
- value.set("{\"attributes\":{\"when\":147147147147}}");
+ value.set("{\"attributes\":{\"when\":1641535200000}}"); // 2022-01-07 06:00 UTC
 Object row = jserde.deserialize(value);
 StructField f0 = rowOI.getStructFieldRef("when");
 Object fieldData = rowOI.getStructFieldData(row, f0);
 long day = 24*3600*1000; // DateWritable represents days not milliseconds.
- long epochExpected = 147147147147L;
+ long epochExpected = 1641535200000L; // or likely 00:00 UTC
  Assert.assertEquals(epochExpected/day, epochFromWritable(fieldData)/day);
- value.set("{\"attributes\":{\"when\":142857142857}}");
+
+ value.set("{\"attributes\":{\"when\":1641578400000}}"); // 2022-01-07 18:00 UTC
  row = jserde.deserialize(value);
 fieldData = rowOI.getStructFieldData(row, f0);
-epochExpected = 142857142857L;
+epochExpected = 1641578400000L; // or likely 00:00 UTC
  Assert.assertEquals(epochExpected/day, epochFromWritable(fieldData)/day);
 }
 

hive/src/test/java/com/esri/hadoop/hive/serde/TestGeoJsonSerDe.java

@@ -54,24 +54,33 @@ public void TestIntWrite() throws Exception {
 
 @Test
 public void TestEpochWrite() throws Exception {
- ArrayList<Object> stuff = new ArrayList<Object>();
 Properties proptab = new Properties();
 proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "when");
 proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
 AbstractSerDe jserde = mkSerDe(proptab);
  StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
+long day = 24*3600*1000; // DateWritable represents days not milliseconds.
 
- // {"properties":{"when":147147147147}}
-long epoch = 147147147147L;
- long zoned = epoch - TimeZone.getDefault().getOffset(epoch);
- java.sql.Date expected = new java.sql.Date(zoned);
- addWritable(stuff, expected);
+long epoch = 1641535200000L; // 2022-01-07 06:00 UTC
+ java.sql.Date jsd = new java.sql.Date(epoch); // zoned?
+ArrayList<Object> stuff = new ArrayList<Object>();
+ addWritable(stuff, jsd);
  Writable jsw = jserde.serialize(stuff, rowOI);
 JsonNode jn = new ObjectMapper().readTree(((Text)jsw).toString());
 jn = jn.findValue("properties");
 jn = jn.findValue("when");
-java.sql.Date actual = new java.sql.Date(jn.getLongValue());
-long day = 24*3600*1000; // DateWritable represents days not milliseconds.
+Assert.assertEquals(epoch/day, jn.getLongValue()/day);
+
+epoch = 1641578400000L; // 2022-01-07 18:00 UTC
+ //long zoned = epoch - TimeZone.getDefault().getOffset(epoch);
+ jsd = new java.sql.Date(epoch); // zoned?
+stuff = new ArrayList<Object>();
+ addWritable(stuff, jsd);
+ jsw = jserde.serialize(stuff, rowOI);
+jn = new ObjectMapper().readTree(((Text)jsw).toString());
+jn = jn.findValue("properties");
+jn = jn.findValue("when");
+ System.err.println(jn);
 Assert.assertEquals(epoch/day, jn.getLongValue()/day);
 }
 
@@ -158,18 +167,20 @@ public void TestEpochParse() throws Exception {
 proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "date");
 jserde.initialize(config, proptab);
  StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
+ // Half a day apart to test both a.m. & p.m. whether in East or West
 
- value.set("{\"properties\":{\"when\":147147147147}}");
+ value.set("{\"properties\":{\"when\":1641535200000}}"); // 2022-01-07 06:00 UTC
 Object row = jserde.deserialize(value);
 StructField f0 = rowOI.getStructFieldRef("when");
 Object fieldData = rowOI.getStructFieldData(row, f0);
 long day = 24*3600*1000; // DateWritable represents days not milliseconds.
- long epochExpected = 147147147147L;
+ long epochExpected = 1641535200000L; // or likely 00:00 UTC
  Assert.assertEquals(epochExpected/day, epochFromWritable(fieldData)/day);
- value.set("{\"properties\":{\"when\":142857142857}}");
+
+ value.set("{\"properties\":{\"when\":1641578400000}}"); // 2022-01-07 18:00 UTC
  row = jserde.deserialize(value);
 fieldData = rowOI.getStructFieldData(row, f0);
-epochExpected = 142857142857L;
+epochExpected = 1641578400000L; // or likely 00:00 UTC
  Assert.assertEquals(epochExpected/day, epochFromWritable(fieldData)/day);
 }