Correct data type codes to match official Apache TSFile definitions
diff --git a/DATA_TYPES.md b/DATA_TYPES.md index b29e3c3..60c08b6 100644 --- a/DATA_TYPES.md +++ b/DATA_TYPES.md
@@ -1,10 +1,10 @@ # IoTDB Data Types Support -This document describes the complete set of IoTDB data types supported by the Node.js client. +This document describes the complete set of IoTDB data types supported by the Node.js client, based on the official Apache TSFile type definitions. ## Supported Data Types -The IoTDB Node.js client now supports all IoTDB data types: +The IoTDB Node.js client supports all standard IoTDB data types as defined in Apache TSFile: | Type Code | Type Name | Description | JavaScript Type | Storage Size | |-----------|-----------|-------------|-----------------|--------------| @@ -14,10 +14,21 @@ | 3 | FLOAT | 32-bit floating point | `number` | 4 bytes | | 4 | DOUBLE | 64-bit floating point | `number` | 8 bytes | | 5 | TEXT | UTF-8 encoded text | `string` | Variable (4-byte length + content) | -| 6 | BLOB | Binary data | `Buffer` | Variable (4-byte length + content) | -| 7 | STRING | UTF-8 encoded string | `string` | Variable (4-byte length + content) | -| 8 | DATE | Date (days since epoch) | `Date` | 4 bytes | -| 9 | TIMESTAMP | Timestamp (milliseconds) | `Date` | 8 bytes | +| 6 | VECTOR | Vector data (not yet implemented) | - | - | +| 7 | UNKNOWN | Unknown type (reserved) | - | - | +| 8 | TIMESTAMP | Timestamp (milliseconds) | `Date` | 8 bytes | +| 9 | DATE | Date (days since epoch) | `Date` | 4 bytes | +| 10 | BLOB | Binary data | `Buffer` | Variable (4-byte length + content) | +| 11 | STRING | UTF-8 encoded string | `string` | Variable (4-byte length + content) | +| 12 | OBJECT | Object type (reserved) | - | - | + +**Note**: Types 0-5, 8-11 are fully supported. Types 6, 7, and 12 are reserved for future use. + +## Reference + +Type definitions are based on: +- [Apache TSFile TSDataType.java](https://github.com/apache/tsfile/blob/develop/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java) +- [Apache IoTDB C# Client Constants](https://github.com/apache/iotdb-client-csharp/blob/main/src/Apache.IoTDB/IoTDBConstants.cs) ## Usage Examples @@ -90,7 +101,7 @@ const tablet = { deviceId: 'root.test.device1', measurements: ['blob_sensor'], - dataTypes: [6], // BLOB + dataTypes: [10], // BLOB (type code 10) timestamps: [Date.now()], values: [ [Buffer.from([0x01, 0x02, 0x03, 0x04])], @@ -109,17 +120,17 @@ console.log(blobData); // <Buffer 01 02 03 04> ``` -### Working with DATE and TIMESTAMP +### Working with TIMESTAMP and DATE ```typescript -// DATE type stores dates without time +// TIMESTAMP stores millisecond precision, DATE stores day precision const tablet = { deviceId: 'root.test.device1', - measurements: ['date_sensor', 'timestamp_sensor'], - dataTypes: [8, 9], // DATE, TIMESTAMP + measurements: ['timestamp_sensor', 'date_sensor'], + dataTypes: [8, 9], // TIMESTAMP=8, DATE=9 timestamps: [Date.now()], values: [ - [new Date('2024-01-15'), new Date('2024-01-15T10:30:00Z')], + [new Date('2024-01-15T10:30:00Z'), new Date('2024-01-15')], ], }; @@ -127,11 +138,11 @@ // Query returns Date objects const result = await session.executeQueryStatement( - 'SELECT date_sensor, timestamp_sensor FROM root.test.device1' + 'SELECT timestamp_sensor, date_sensor FROM root.test.device1' ); -const dateValue = result.rows[0][1]; // Date object -const timestampValue = result.rows[0][2]; // Date object +const timestampValue = result.rows[0][1]; // Date object with time +const dateValue = result.rows[0][2]; // Date object (day precision) ``` ### Query Results with All Types
diff --git a/DATA_TYPES_IMPLEMENTATION_SUMMARY.md b/DATA_TYPES_IMPLEMENTATION_SUMMARY.md index c4a6adc..7b45224 100644 --- a/DATA_TYPES_IMPLEMENTATION_SUMMARY.md +++ b/DATA_TYPES_IMPLEMENTATION_SUMMARY.md
@@ -1,18 +1,19 @@ # Data Types Enhancement Summary -## User Request (Comment #3714178397) +## User Request (Comment #3714178397 & #3714232108) User @CritasWang requested: 1. Confirm all IoTDB data types are properly handled, including BLOB, STRING, DATE, TIMESTAMP 2. Design a test case that simultaneously supports all data types +3. **IMPORTANT**: Use official Apache TSFile type enumerations, not made-up values -## Implementation +## Implementation - CORRECTED -### Added Support for 4 New Data Types +### Supported Data Types Based on Apache TSFile -Extended the IoTDB Node.js client to support all 10 data types: +The IoTDB Node.js client now supports all standard IoTDB data types as defined in Apache TSFile: -**Previously Supported (0-5):** +**Core Types (0-5):** - BOOLEAN (0) - INT32 (1) - INT64 (2) @@ -20,32 +21,30 @@ - DOUBLE (4) - TEXT (5) -**Newly Added (6-9):** -- BLOB (6): Binary data -- STRING (7): UTF-8 string (similar to TEXT) -- DATE (8): Date values (days since epoch) -- TIMESTAMP (9): Timestamp values (milliseconds) +**Reserved Types:** +- VECTOR (6): Reserved for future use +- UNKNOWN (7): Reserved + +**Extended Types (8-11):** +- TIMESTAMP (8): Timestamp values (milliseconds) +- DATE (9): Date values (days since epoch) +- BLOB (10): Binary data +- STRING (11): UTF-8 string (similar to TEXT) + +**Reserved:** +- OBJECT (12): Reserved for future use ### Code Changes #### 1. Serialization (Session.ts - `serializeColumn`) -Added support for writing new data types to IoTDB: +Corrected type codes based on official Apache TSFile definitions: ```typescript -case 6: { // BLOB - const blobBuffers = values.map((v) => { - const blob = Buffer.isBuffer(v) ? v : Buffer.from(v); - const len = Buffer.alloc(4); - len.writeInt32LE(blob.length); - return Buffer.concat([len, blob]); - }); - return Buffer.concat(blobBuffers); -} - -case 7: // STRING (same as TEXT) -case 8: // DATE (INT32 - days since epoch) -case 9: // TIMESTAMP (INT64 - milliseconds) +case 8: { // TIMESTAMP (INT64 - milliseconds) +case 9: { // DATE (INT32 - days since epoch) +case 10: { // BLOB +case 11: // STRING (same as TEXT) ``` #### 2. Deserialization (Session.ts - `deserializeColumn`) @@ -116,10 +115,13 @@ | FLOAT | 3 | number | number | 4 bytes | | DOUBLE | 4 | number | number | 8 bytes | | TEXT | 5 | string | string | Variable | -| BLOB | 6 | Buffer | Buffer | Variable | -| STRING | 7 | string | string | Variable | -| DATE | 8 | Date/number | Date | 4 bytes | -| TIMESTAMP | 9 | Date/bigint | Date | 8 bytes | +| VECTOR | 6 | - | - | - (reserved) | +| UNKNOWN | 7 | - | - | - (reserved) | +| TIMESTAMP | 8 | Date/bigint | Date | 8 bytes | +| DATE | 9 | Date/number | Date | 4 bytes | +| BLOB | 10 | Buffer | Buffer | Variable | +| STRING | 11 | string | string | Variable | +| OBJECT | 12 | - | - | - (reserved) | ### Usage Example @@ -135,9 +137,9 @@ 'float_sensor', 'double_sensor', 'text_sensor', - // Could add: 'blob_sensor', 'string_sensor', 'date_sensor', 'timestamp_sensor' + // Extended types: 'timestamp_sensor', 'date_sensor', 'blob_sensor', 'string_sensor' ], - dataTypes: [0, 1, 2, 3, 4, 5], // All 10 types supported + dataTypes: [0, 1, 2, 3, 4, 5], // Core types: BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT timestamps: [now, now + 1, now + 2], values: [ [true, 100, 1000n, 1.23, 4.56, 'hello'],
diff --git a/src/client/Session.ts b/src/client/Session.ts index 59910d8..7f48bf7 100644 --- a/src/client/Session.ts +++ b/src/client/Session.ts
@@ -185,8 +185,9 @@ } private serializeColumn(values: any[], dataType: number): Buffer { - // TSDataType: BOOLEAN(0), INT32(1), INT64(2), FLOAT(3), DOUBLE(4), TEXT(5), - // BLOB(6), STRING(7), DATE(8), TIMESTAMP(9) + // TSDataType from Apache TSFile: + // BOOLEAN(0), INT32(1), INT64(2), FLOAT(3), DOUBLE(4), TEXT(5), + // VECTOR(6), UNKNOWN(7), TIMESTAMP(8), DATE(9), BLOB(10), STRING(11), OBJECT(12) switch (dataType) { case 0: // BOOLEAN return Buffer.from(values.map((v) => (v ? 1 : 0))); @@ -201,7 +202,7 @@ return Buffer.from(new Float64Array(values).buffer); } case 5: // TEXT - case 7: { // STRING (similar to TEXT) + case 11: { // STRING (similar to TEXT) const strBuffers = values.map((v) => { const str = String(v); const len = Buffer.alloc(4); @@ -210,16 +211,15 @@ }); return Buffer.concat(strBuffers); } - case 6: { // BLOB - const blobBuffers = values.map((v) => { - const blob = Buffer.isBuffer(v) ? v : Buffer.from(v); - const len = Buffer.alloc(4); - len.writeInt32LE(blob.length); - return Buffer.concat([len, blob]); - }); - return Buffer.concat(blobBuffers); + case 8: { // TIMESTAMP (stored as INT64 - milliseconds) + return Buffer.from(new BigInt64Array(values.map(v => { + if (v instanceof Date) { + return BigInt(v.getTime()); + } + return BigInt(v); + })).buffer); } - case 8: { // DATE (stored as INT32 - days since epoch) + case 9: { // DATE (stored as INT32 - days since epoch) return Buffer.from(new Int32Array(values.map(v => { if (v instanceof Date) { return Math.floor(v.getTime() / (24 * 60 * 60 * 1000)); @@ -227,13 +227,14 @@ return v; })).buffer); } - case 9: { // TIMESTAMP (stored as INT64 - milliseconds) - return Buffer.from(new BigInt64Array(values.map(v => { - if (v instanceof Date) { - return BigInt(v.getTime()); - } - return BigInt(v); - })).buffer); + case 10: { // BLOB + const blobBuffers = values.map((v) => { + const blob = Buffer.isBuffer(v) ? v : Buffer.from(v); + const len = Buffer.alloc(4); + len.writeInt32LE(blob.length); + return Buffer.concat([len, blob]); + }); + return Buffer.concat(blobBuffers); } default: throw new Error(`Unsupported data type: ${dataType}`); @@ -364,10 +365,10 @@ else if (typeStr.includes('FLOAT')) dataType = 3; else if (typeStr.includes('DOUBLE')) dataType = 4; else if (typeStr.includes('TEXT')) dataType = 5; - else if (typeStr.includes('BLOB')) dataType = 6; - else if (typeStr.includes('STRING')) dataType = 7; - else if (typeStr.includes('DATE')) dataType = 8; - else if (typeStr.includes('TIMESTAMP')) dataType = 9; + else if (typeStr.includes('TIMESTAMP')) dataType = 8; + else if (typeStr.includes('DATE')) dataType = 9; + else if (typeStr.includes('BLOB')) dataType = 10; + else if (typeStr.includes('STRING')) dataType = 11; } logger.debug(`parseDataSet: column ${colIndex}, dataType = ${dataType}, valueBuffer.length = ${valueBuffer.length}`); @@ -457,7 +458,7 @@ break; } case 5: // TEXT - case 7: { // STRING (similar to TEXT) + case 11: { // STRING (similar to TEXT) let offset = 0; for (let i = 0; i < rowCount && offset < buffer.length; i++) { if (this.isNull(bitmap, i)) { @@ -474,24 +475,21 @@ } break; } - case 6: { // BLOB - let offset = 0; - for (let i = 0; i < rowCount && offset < buffer.length; i++) { + case 8: { // TIMESTAMP (stored as INT64 - milliseconds) + const bigInt64Array = new BigInt64Array(buffer.buffer, buffer.byteOffset, Math.floor(buffer.length / 8)); + for (let i = 0; i < rowCount && i < bigInt64Array.length; i++) { if (this.isNull(bitmap, i)) { values.push(null); } else { - if (offset + 4 > buffer.length) break; - const blobLength = buffer.readInt32LE(offset); - offset += 4; - if (offset + blobLength > buffer.length) break; - const blob = buffer.slice(offset, offset + blobLength); - values.push(blob); - offset += blobLength; + // Convert milliseconds to Date object + const timestamp = Number(bigInt64Array[i]); + const date = new Date(timestamp); + values.push(date); } } break; } - case 8: { // DATE (stored as INT32 - days since epoch) + case 9: { // DATE (stored as INT32 - days since epoch) const int32Array = new Int32Array(buffer.buffer, buffer.byteOffset, Math.floor(buffer.length / 4)); for (let i = 0; i < rowCount && i < int32Array.length; i++) { if (this.isNull(bitmap, i)) { @@ -505,16 +503,19 @@ } break; } - case 9: { // TIMESTAMP (stored as INT64 - milliseconds) - const bigInt64Array = new BigInt64Array(buffer.buffer, buffer.byteOffset, Math.floor(buffer.length / 8)); - for (let i = 0; i < rowCount && i < bigInt64Array.length; i++) { + case 10: { // BLOB + let offset = 0; + for (let i = 0; i < rowCount && offset < buffer.length; i++) { if (this.isNull(bitmap, i)) { values.push(null); } else { - // Convert milliseconds to Date object - const timestamp = Number(bigInt64Array[i]); - const date = new Date(timestamp); - values.push(date); + if (offset + 4 > buffer.length) break; + const blobLength = buffer.readInt32LE(offset); + offset += 4; + if (offset + blobLength > buffer.length) break; + const blob = buffer.slice(offset, offset + blobLength); + values.push(blob); + offset += blobLength; } } break;
diff --git a/tests/e2e/AllDataTypes.test.ts b/tests/e2e/AllDataTypes.test.ts index d672c4b..640a874 100644 --- a/tests/e2e/AllDataTypes.test.ts +++ b/tests/e2e/AllDataTypes.test.ts
@@ -115,7 +115,8 @@ 'double_sensor', 'text_sensor' ], - // Data types: BOOLEAN(0), INT32(1), INT64(2), FLOAT(3), DOUBLE(4), TEXT(5) + // Data types from Apache TSFile: + // BOOLEAN(0), INT32(1), INT64(2), FLOAT(3), DOUBLE(4), TEXT(5) dataTypes: [0, 1, 2, 3, 4, 5], timestamps: [now, now + 1, now + 2], values: [