Dataset Management APIs
List of APIs to manage datasets
Last updated
Was this helpful?
List of APIs to manage datasets
Last updated
Was this helpful?
This API allows you to read dataset from the requested dataset_id. User can request for the specific fields and status of the dataset through the request params. By default, the API returns the dataset of status "Live". This API accepts the parameter mode=edit to read the draft dataset. If a draft dataset is not found, it creates one using the live dataset and returns the dataset details.
Unique identifier for the dataset
editconnect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4curl -L \
--url 'http://localhost:3000/v2/datasets/read/{dataset_id}'{
"id": "api.datasets.read",
"ver": "v2",
"ts": "2024-07-17T17:38:55+05:30",
"params": {
"status": "SUCCESS",
"resmsgid": "8c8a2852-54bc-43fb-b063-7f359d11930a"
},
"responseCode": "OK",
"result": {
"dataset_id": "master-test",
"name": "master-test",
"type": "master",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "userid",
"timestamp_key": "",
"exclude_fields": [],
"entry_topic": "local.masterdata.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 54
}
}
}OK
This API allows you to create new datasets used by the analytical data source.
application/json{"id":"api.datasets.create","ver":"v2","ts":"2024-04-10T16:10:50+05:30","params":{"msgid":"4a7f14c3-d61e-4d4f-be78-181834eeff6d"},"request":{"dataset_id":"telemetry_record-t4","type":"event","name":"sb-telemetry","validation_config":{"validate":true,"mode":"Strict"},"extraction_config":{"is_batch_event":true,"extraction_key":"events","dedup_config":{"drop_duplicates":true,"dedup_key":"id"}},"dedup_config":{"drop_duplicates":true,"dedup_key":"mid"},"data_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"mid":{"type":"string","arrival_format":"text","data_type":"string"},"ets":{"type":"integer","arrival_format":"number","data_type":"epoch"},"eid":{"type":"string","arrival_format":"text","data_type":"string"}},"additionalProperties":true},"denorm_config":{"denorm_fields":[{"denorm_key":"eid","denorm_out_field":"userdata","dataset_id":"master-telemetry"}]},"transformations_config":[{"field_key":"email","transformation_function":{"type":"mask","expr":"mid","datatype":"string","category":"pii"},"mode":"Strict"}],"tags":["tag1"]}}curl -L \
--request POST \
--url 'http://localhost:3000/v2/datasets/create' \
--header 'Content-Type: application/json' \
--data '{
"id": "api.datasets.create",
"ver": "v2",
"ts": "2024-04-10T16:10:50+05:30",
"params": {
"msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d"
},
"request": {
"dataset_id": "telemetry_record-t4",
"type": "event",
"name": "sb-telemetry",
"validation_config": {
"validate": true,
"mode": "Strict"
},
"extraction_config": {
"is_batch_event": true,
"extraction_key": "events",
"dedup_config": {
"drop_duplicates": true,
"dedup_key": "id"
}
},
"dedup_config": {
"drop_duplicates": true,
"dedup_key": "mid"
},
"data_schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"mid": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"ets": {
"type": "integer",
"arrival_format": "number",
"data_type": "epoch"
},
"eid": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
}
},
"additionalProperties": true
},
"denorm_config": {
"denorm_fields": [
{
"denorm_key": "eid",
"denorm_out_field": "userdata",
"dataset_id": "master-telemetry"
}
]
},
"transformations_config": [
{
"field_key": "email",
"transformation_function": {
"type": "mask",
"expr": "mid",
"datatype": "string",
"category": "pii"
},
"mode": "Strict"
}
],
"tags": [
"tag1"
]
}
}'{
"id": "api.datasets.create",
"ver": "v2",
"ts": "2024-07-15T18:44:08+05:30",
"params": {
"status": "SUCCESS",
"msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d",
"resmsgid": "276c042c-0f23-4b26-9b10-6fe48bbc2d3a"
},
"responseCode": "OK",
"result": {
"id": "telemetry_record-t4",
"version_key": "1721049248930"
}
}OK
This API allows you to update existing datasets, add or remove denorm fields used by the analytical data source. User can even add, remove or update transformations and connectors
application/json{"id":"api.datasets.update","ver":"v2","ts":"2024-04-10T16:10:50+05:30","params":{"msgid":"4a7f14c3-d61e-4d4f-be78-181834eeff6d"},"request":{"dataset_id":"telemetry_record-t4","version_key":"1721135455988","name":"sb-telemetry","validation_config":{"validate":true,"mode":"Strict"},"extraction_config":{"is_batch_event":true,"extraction_key":"events","dedup_config":{"drop_duplicates":true,"dedup_key":"ipid"}},"dedup_config":{"drop_duplicates":true,"dedup_key":"mid"},"data_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"midpid":{"type":"string","arrival_format":"text","data_type":"string"},"miduwi":{"type":"integer","arrival_format":"number","data_type":"epoch"},"mid":{"type":"string","arrival_format":"text","data_type":"string"},"sid":{"type":"string","arrival_format":"text","data_type":"string"}},"additionalProperties":true},"denorm_config":{"denorm_fields":[{"value":{"denorm_key":"eid","denorm_out_field":"userdata"},"action":"remove"},{"value":{"denorm_key":"eid","denorm_out_field":"edata","dataset_id":"trip-details"},"action":"upsert"}]},"transformations_config":[{"value":{"field_key":"email","transformation_function":{"type":"mask","expr":"mid","datatype":"string","category":"pii"},"mode":"Strict"},"action":"upsert"},{"value":{"field_key":"email_id","transformation_function":{"type":"mask","expr":"mid","datatype":"string","category":"pii"},"mode":"Strict"},"action":"remove"}],"tags":[],"connectors_config":[{"value":{"connector_id":"jdbc","connector_config":{"source_database_type":"postgresql","source_database_host":"postgresql-hl.postgresql.svc.cluster.local.master","source_database_port":5432,"source_database_name":"obsrv_sample_datasets_1","source_database_username":"postgres","source_database_pwd":"postgres","table":"new_york_taxi_data","timestamp-column":"tpep_pickup_datetime","batch-size":100,"max-batches":2},"operations_config":{"polling_interval":"periodic","schedule":"twice"}},"action":"upsert"}]}}curl -L \
--request PATCH \
--url 'http://localhost:3000/v2/datasets/update' \
--header 'Content-Type: application/json' \
--data '{
"id": "api.datasets.update",
"ver": "v2",
"ts": "2024-04-10T16:10:50+05:30",
"params": {
"msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d"
},
"request": {
"dataset_id": "telemetry_record-t4",
"version_key": "1721135455988",
"name": "sb-telemetry",
"validation_config": {
"validate": true,
"mode": "Strict"
},
"extraction_config": {
"is_batch_event": true,
"extraction_key": "events",
"dedup_config": {
"drop_duplicates": true,
"dedup_key": "ipid"
}
},
"dedup_config": {
"drop_duplicates": true,
"dedup_key": "mid"
},
"data_schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"midpid": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"miduwi": {
"type": "integer",
"arrival_format": "number",
"data_type": "epoch"
},
"mid": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"sid": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
}
},
"additionalProperties": true
},
"denorm_config": {
"denorm_fields": [
{
"value": {
"denorm_key": "eid",
"denorm_out_field": "userdata"
},
"action": "remove"
},
{
"value": {
"denorm_key": "eid",
"denorm_out_field": "edata",
"dataset_id": "trip-details"
},
"action": "upsert"
}
]
},
"transformations_config": [
{
"value": {
"field_key": "email",
"transformation_function": {
"type": "mask",
"expr": "mid",
"datatype": "string",
"category": "pii"
},
"mode": "Strict"
},
"action": "upsert"
},
{
"value": {
"field_key": "email_id",
"transformation_function": {
"type": "mask",
"expr": "mid",
"datatype": "string",
"category": "pii"
},
"mode": "Strict"
},
"action": "remove"
}
],
"tags": [],
"connectors_config": [
{
"value": {
"connector_id": "jdbc",
"connector_config": {
"source_database_type": "postgresql",
"source_database_host": "postgresql-hl.postgresql.svc.cluster.local.master",
"source_database_port": 5432,
"source_database_name": "obsrv_sample_datasets_1",
"source_database_username": "postgres",
"source_database_pwd": "postgres",
"table": "new_york_taxi_data",
"timestamp-column": "tpep_pickup_datetime",
"batch-size": 100,
"max-batches": 2
},
"operations_config": {
"polling_interval": "periodic",
"schedule": "twice"
}
},
"action": "upsert"
}
]
}
}'{
"id": "api.datasets.update",
"ver": "v2",
"ts": "2024-07-16T18:30:45+05:30",
"params": {
"status": "SUCCESS",
"msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d",
"resmsgid": "354f1fec-0c39-42ee-a52a-49552f847c11"
},
"responseCode": "OK",
"result": {
"message": "Dataset is updated successfully",
"id": "telemetry_record-t4",
"version_key": "1721134845559"
}
}OK
This API allows you to list all datasets. User can apply filters on dataset status and type.
{"id":"api.datasets.list","ver":"v2","ts":"2024-04-10T16:10:50+05:30","params":{"msgid":"4a7f14c3-d61e-4d4f-be78-181834eeff6d"},"request":{"filters":{"status":["Live"]}}}curl -L \
--request POST \
--url 'http://localhost:3000/v2/datasets/list' \
--header 'Content-Type: application/json' \
--data '{
"id": "api.datasets.list",
"ver": "v2",
"ts": "2024-04-10T16:10:50+05:30",
"params": {
"msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d"
},
"request": {
"filters": {
"status": [
"Live"
]
}
}
}'{
"id": "api.datasets.list",
"ver": "v2",
"ts": "2024-07-17T17:55:36+05:30",
"params": {
"status": "SUCCESS",
"msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d",
"resmsgid": "97efe04d-e981-493d-9ee7-a6dad6887d64"
},
"responseCode": "OK",
"result": {
"data": [
{
"dataset_id": "telemetry-summary",
"name": "telemetry-summary",
"type": "event",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "",
"timestamp_key": "obsrv_meta.syncts",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "tripdetailstest",
"name": "TripDetailsTest1",
"type": "event",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "",
"timestamp_key": "tpep_dropoff_datetime",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "test-normal",
"name": "test-normal-renamed",
"type": "event",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "",
"timestamp_key": "tpep_pickup_datetime",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "test-dataset",
"name": "test-dataset-renamed",
"type": "event",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "",
"timestamp_key": "tpep_dropoff_datetime",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "triptestdataset",
"name": "triptestdataset",
"type": "event",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "",
"timestamp_key": "tpep_pickup_datetime",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "master-test",
"name": "master-test",
"type": "master",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "userid",
"timestamp_key": "",
"exclude_fields": [],
"entry_topic": "local.masterdata.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 54
}
},
{
"dataset_id": "test-trip-details",
"name": "test-trip-details",
"type": "event",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "",
"timestamp_key": "obsrv_meta.syncts",
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0,
"file_upload_path": []
}
},
{
"dataset_id": "sb-telemetry",
"name": "sb-telemetry",
"type": "event",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "",
"timestamp_key": "ets",
"exclude_fields": [],
"entry_topic": "sb-dev.ingest",
"redis_db_host": "obsrv-redis-master.redis.svc.cluster.local",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "sb-telemetry-user",
"name": "sb-telemetry-user",
"type": "master",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "id",
"timestamp_key": "",
"exclude_fields": [],
"entry_topic": "sb-dev.masterdata.ingest",
"redis_db_host": "obsrv-redis-master.redis.svc.cluster.local",
"redis_db_port": 6379,
"index_data": false,
"redis_db": 4
}
},
{
"dataset_id": "sb-telemetry-test",
"name": "sb-telemetry",
"type": "event",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "",
"timestamp_key": "ets",
"exclude_fields": [],
"entry_topic": "sb-dev.ingest",
"redis_db_host": "obsrv-redis-master.redis.svc.cluster.local",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "test-changes",
"name": "test-changes",
"type": "event",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "",
"timestamp_key": "date",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "sample1",
"name": "sample1",
"type": "event",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "",
"timestamp_key": "time",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "telemetry-events",
"name": "telemetry-events",
"type": "event",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "",
"timestamp_key": "date",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "taxt_trip",
"name": "taxt_trip",
"type": "event",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "",
"timestamp_key": "date",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "test",
"name": "test",
"type": "event",
"status": "Live",
"tags": [
"TAG1"
],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "",
"timestamp_key": "obsrv_meta.syncts",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "beckn-test-data",
"name": "beckn-test-data",
"type": "event",
"status": "Live",
"tags": [],
"version": 1,
"api_version": "v1",
"dataset_config": {
"data_key": "",
"timestamp_key": "ets",
"exclude_fields": [],
"entry_topic": "beckn-test-data",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "telemetry_record-t4",
"name": "sb-telemetry",
"type": "event",
"status": "Draft",
"tags": [
"tag1"
],
"version": 1,
"api_version": "v2",
"dataset_config": {
"indexing_config": {
"olap_store_enabled": true,
"lakehouse_enabled": true,
"cache_enabled": false
},
"keys_config": {
"data_key": "",
"partition_key": "",
"timestamp_key": "obsrv_meta.syncts"
},
"cache_config": {
"redis_db_port": null,
"redis_db": 0
},
"file_upload_path": []
}
},
{
"dataset_id": "telemetry_events",
"name": "sb-telemetry",
"type": "event",
"status": "Draft",
"tags": [],
"version": 1,
"api_version": "v2",
"dataset_config": {
"indexing_config": {
"olap_store_enabled": true,
"lakehouse_enabled": true,
"cache_enabled": false
},
"keys_config": {
"data_key": "",
"partition_key": "",
"timestamp_key": "obsrv_meta.syncts"
},
"cache_config": {
"redis_db_port": null,
"redis_db": 0
},
"file_upload_path": []
}
},
{
"dataset_id": "telemetry_record-master",
"name": "sb-telemetry",
"type": "master",
"status": "Draft",
"tags": [
"tag1"
],
"version": 1,
"api_version": "v2",
"dataset_config": {
"indexing_config": {
"olap_store_enabled": true,
"lakehouse_enabled": true,
"cache_enabled": false
},
"keys_config": {
"data_key": "",
"partition_key": "",
"timestamp_key": "obsrv_meta.syncts"
},
"cache_config": {
"redis_db_port": null,
"redis_db": 0
},
"file_upload_path": []
}
},
{
"dataset_id": "generate-schema",
"name": "generate-schema",
"type": "event",
"status": "Draft",
"tags": [],
"version": 1,
"api_version": null,
"dataset_config": {
"data_key": "",
"timestamp_key": "",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "test-summary",
"name": "test-summary",
"type": "event",
"status": "ReadyToPublish",
"tags": [],
"version": 1,
"api_version": null,
"dataset_config": {
"data_key": "",
"timestamp_key": "obsrv_meta.syncts",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "trip-details1",
"name": "trip-details",
"type": "event",
"status": "ReadyToPublish",
"tags": [],
"version": 1,
"api_version": null,
"dataset_config": {
"data_key": "",
"timestamp_key": "tpep_pickup_datetime",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "telemetry-test-dataset",
"name": "telemetry-test-dataset",
"type": "event",
"status": "ReadyToPublish",
"tags": [],
"version": 1,
"api_version": null,
"dataset_config": {
"data_key": "",
"timestamp_key": "obsrv_meta.syncts",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "trip-test",
"name": "trip-test",
"type": "event",
"status": "Draft",
"tags": [],
"version": 1,
"api_version": null,
"dataset_config": {
"data_key": "",
"timestamp_key": "",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "sample-trip-details",
"name": "sample-trip-details",
"type": "event",
"status": "Draft",
"tags": [],
"version": 1,
"api_version": null,
"dataset_config": {
"data_key": "",
"timestamp_key": "",
"exclude_fields": [],
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0
}
},
{
"dataset_id": "test-rollup",
"name": "test-rollup",
"type": "event",
"status": "ReadyToPublish",
"tags": [],
"version": 1,
"api_version": "v2",
"dataset_config": {
"data_key": "",
"timestamp_key": "obsrv_meta.syncts",
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0,
"file_upload_path": [],
"dataMappings": {
"text": {
"arrival_format": [
"string"
],
"store_format": {
"string": {
"jsonSchema": "string",
"datasource": "string"
},
"date-time": {
"jsonSchema": "string",
"datasource": "string"
},
"date": {
"jsonSchema": "string",
"datasource": "string"
},
"boolean": {
"jsonSchema": "string",
"datasource": "boolean"
},
"epoch": {
"jsonSchema": "string",
"datasource": "integer"
},
"long": {
"jsonSchema": "string",
"datasource": "long"
},
"double": {
"jsonSchema": "string",
"datasource": "double"
},
"bigdecimal": {
"jsonSchema": "string",
"datasource": "double"
},
"integer": {
"jsonSchema": "string",
"datasource": "long"
}
}
},
"number": {
"arrival_format": [
"number",
"integer"
],
"store_format": {
"integer": {
"jsonSchema": "integer",
"datasource": "long"
},
"float": {
"jsonSchema": "number",
"datasource": "double"
},
"long": {
"jsonSchema": "integer",
"datasource": "long"
},
"double": {
"jsonSchema": "number",
"datasource": "double"
},
"bigdecimal": {
"jsonSchema": "number",
"datasource": "double"
},
"epoch": {
"jsonSchema": "integer",
"datasource": "long"
},
"number": {
"jsonSchema": "number",
"datasource": "double"
}
}
},
"object": {
"arrival_format": [
"object"
],
"store_format": {
"object": {
"jsonSchema": "object",
"datasource": "json"
}
}
},
"array": {
"arrival_format": [
"array"
],
"store_format": {
"array": {
"jsonSchema": "array",
"datasource": "array"
}
}
},
"boolean": {
"arrival_format": [
"boolean"
],
"store_format": {
"boolean": {
"jsonSchema": "boolean",
"datasource": "boolean"
}
}
}
},
"configurations": {
"indexConfiguration": {
"index": {
"Event Arrival Time": "obsrv_meta.syncts"
},
"rollupSuggestions": {}
},
"processing": {
"dedupKeys": [],
"dropDuplicates": [
"Yes",
"No"
]
}
},
"mergedEvent": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"eid": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"ets": {
"type": "integer",
"suggestions": [
{
"message": "The Property 'ets' appears to be 'epoch' format type.",
"severity": "",
"path": "properties.ets"
}
],
"arrival_format": "number",
"data_type": "epoch"
},
"syncts": {
"type": "integer",
"suggestions": [
{
"message": "The Property 'syncts' appears to be 'epoch' format type.",
"severity": "",
"path": "properties.syncts"
}
],
"arrival_format": "number",
"data_type": "epoch"
},
"ver": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"mid": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"uid": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"content_id": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"context": {
"type": "object",
"properties": {
"pdata": {
"type": "object",
"properties": {
"id": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"ver": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"model": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"pid": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
}
},
"arrival_format": "object",
"data_type": "object",
"additionalProperties": true
},
"granularity": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"date_range": {
"type": "object",
"properties": {
"from": {
"type": "integer",
"suggestions": [
{
"message": "The Property 'context.date_range.from' appears to be 'epoch' format type.",
"severity": "",
"path": "properties.context.properties.date_range.properties.from"
}
],
"arrival_format": "number",
"data_type": "epoch"
},
"to": {
"type": "integer",
"suggestions": [
{
"message": "The Property 'context.date_range.to' appears to be 'epoch' format type.",
"severity": "",
"path": "properties.context.properties.date_range.properties.to"
}
],
"arrival_format": "number",
"data_type": "epoch"
}
},
"arrival_format": "object",
"data_type": "object",
"additionalProperties": true
},
"rollup": {
"type": "object",
"arrival_format": "object",
"data_type": "object",
"additionalProperties": true
},
"cdata": {
"type": "array",
"items": {
"type": "object",
"properties": {
"key": {
"type": "string"
},
"value": {
"type": "string"
}
}
},
"arrival_format": "array",
"data_type": "array"
}
},
"arrival_format": "object",
"data_type": "object",
"additionalProperties": true
},
"dimensions": {
"type": "object",
"properties": {
"did": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"pdata": {
"type": "object",
"properties": {
"id": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"ver": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"pid": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
}
},
"arrival_format": "object",
"data_type": "object",
"additionalProperties": true
},
"sid": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"channel": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"type": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"mode": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"content_type": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
}
},
"arrival_format": "object",
"data_type": "object",
"additionalProperties": true
},
"edata": {
"type": "object",
"properties": {
"eks": {
"type": "object",
"properties": {
"interact_events_per_min": {
"type": "integer",
"arrival_format": "number",
"data_type": "integer"
},
"start_time": {
"type": "integer",
"suggestions": [
{
"message": "The Property 'edata.eks.start_time' appears to be 'epoch' format type.",
"severity": "",
"path": "properties.edata.properties.eks.properties.start_time"
}
],
"arrival_format": "number",
"data_type": "epoch"
},
"interact_events_count": {
"type": "integer",
"arrival_format": "number",
"data_type": "integer"
},
"item_responses": {
"type": "array",
"items": {
"type": "object",
"properties": {
"key": {
"type": "string"
},
"value": {
"type": "string"
}
}
},
"arrival_format": "array",
"data_type": "array"
},
"end_time": {
"type": "integer",
"suggestions": [
{
"message": "The Property 'edata.eks.end_time' appears to be 'epoch' format type.",
"severity": "",
"path": "properties.edata.properties.eks.properties.end_time"
}
],
"arrival_format": "number",
"data_type": "epoch"
},
"events_summary": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"count": {
"type": "integer",
"arrival_format": "number",
"data_type": "integer"
}
},
"additionalProperties": true
},
"arrival_format": "array",
"data_type": "array"
},
"page_summary": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"env": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"time_spent": {
"type": "integer",
"arrival_format": "number",
"data_type": "integer"
},
"visit_count": {
"type": "integer",
"arrival_format": "number",
"data_type": "integer"
}
},
"additionalProperties": true
},
"arrival_format": "array",
"data_type": "array"
},
"time_diff": {
"type": "number",
"arrival_format": "number",
"data_type": "number"
},
"telemetry_version": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"env_summary": {
"type": "array",
"items": {
"type": "object",
"properties": {
"env": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"time_spent": {
"type": "integer",
"arrival_format": "number",
"data_type": "integer"
},
"count": {
"type": "integer",
"arrival_format": "number",
"data_type": "integer"
}
},
"additionalProperties": true
},
"arrival_format": "array",
"data_type": "array"
},
"time_spent": {
"type": "number",
"arrival_format": "number",
"data_type": "number"
}
},
"arrival_format": "object",
"data_type": "object",
"additionalProperties": true
}
},
"arrival_format": "object",
"data_type": "object",
"additionalProperties": true
},
"tags": {
"type": "array",
"items": {
"type": "string"
},
"arrival_format": "array",
"data_type": "array"
},
"object": {
"type": "object",
"properties": {
"ver": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"id": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
}
},
"arrival_format": "object",
"data_type": "object",
"additionalProperties": true
}
},
"additionalProperties": true
}
}
},
{
"dataset_id": "trip",
"name": "trip",
"type": "event",
"status": "ReadyToPublish",
"tags": [],
"version": 1,
"api_version": "v2",
"dataset_config": {
"data_key": "",
"timestamp_key": "obsrv_meta.syncts",
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0,
"file_upload_path": [],
"dataMappings": {
"text": {
"arrival_format": [
"string"
],
"store_format": {
"string": {
"jsonSchema": "string",
"datasource": "string"
},
"date-time": {
"jsonSchema": "string",
"datasource": "string"
},
"date": {
"jsonSchema": "string",
"datasource": "string"
},
"boolean": {
"jsonSchema": "string",
"datasource": "boolean"
},
"epoch": {
"jsonSchema": "string",
"datasource": "integer"
},
"long": {
"jsonSchema": "string",
"datasource": "long"
},
"double": {
"jsonSchema": "string",
"datasource": "double"
},
"bigdecimal": {
"jsonSchema": "string",
"datasource": "double"
},
"integer": {
"jsonSchema": "string",
"datasource": "long"
}
}
},
"number": {
"arrival_format": [
"number",
"integer"
],
"store_format": {
"integer": {
"jsonSchema": "integer",
"datasource": "long"
},
"float": {
"jsonSchema": "number",
"datasource": "double"
},
"long": {
"jsonSchema": "integer",
"datasource": "long"
},
"double": {
"jsonSchema": "number",
"datasource": "double"
},
"bigdecimal": {
"jsonSchema": "number",
"datasource": "double"
},
"epoch": {
"jsonSchema": "integer",
"datasource": "long"
},
"number": {
"jsonSchema": "number",
"datasource": "double"
}
}
},
"object": {
"arrival_format": [
"object"
],
"store_format": {
"object": {
"jsonSchema": "object",
"datasource": "json"
}
}
},
"array": {
"arrival_format": [
"array"
],
"store_format": {
"array": {
"jsonSchema": "array",
"datasource": "array"
}
}
},
"boolean": {
"arrival_format": [
"boolean"
],
"store_format": {
"boolean": {
"jsonSchema": "boolean",
"datasource": "boolean"
}
}
}
},
"configurations": {
"indexConfiguration": {
"index": {
"Event Arrival Time": "obsrv_meta.syncts"
},
"rollupSuggestions": {
"summary": {
"tripID": {
"path": "$.tripID",
"cardinality": 99,
"index": false
}
}
}
},
"processing": {
"dedupKeys": [
"tripID"
],
"dropDuplicates": [
"Yes",
"No"
]
}
},
"mergedEvent": {
"tripID": "02e07922-e8a5-4655-84a8-b5ba1866f9fe",
"VendorID": "2",
"tpep_pickup_datetime": "2023-04-28 00:18:42",
"tpep_dropoff_datetime": "2024-02-15 00:24:38",
"passenger_count": "1",
"trip_distance": "1.60",
"RatecodeID": "1",
"store_and_fwd_flag": "N",
"PULocationID": "236",
"DOLocationID": "239",
"payment_type": "2",
"primary_passenger": {
"email": "Dewayne_Kuvalis17@gmail.com",
"mobile": "1-429-628-3797 x14211"
},
"fare_details": {
"fare_amount": "7",
"extra": "0.5",
"mta_tax": "0.5",
"tip_amount": "0",
"tolls_amount": "0",
"improvement_surcharge": "0.3",
"total_amount": "8.3",
"congestion_surcharge": ""
},
"passenger-name": "yashashk"
}
}
},
{
"dataset_id": "test1",
"name": "test1",
"type": "event",
"status": "ReadyToPublish",
"tags": [],
"version": 1,
"api_version": "v2",
"dataset_config": {
"data_key": "",
"timestamp_key": "obsrv_meta.syncts",
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0,
"file_upload_path": [],
"dataMappings": {
"text": {
"arrival_format": [
"string"
],
"store_format": {
"string": {
"jsonSchema": "string",
"datasource": "string"
},
"date-time": {
"jsonSchema": "string",
"datasource": "string"
},
"date": {
"jsonSchema": "string",
"datasource": "string"
},
"boolean": {
"jsonSchema": "string",
"datasource": "boolean"
},
"epoch": {
"jsonSchema": "string",
"datasource": "integer"
},
"long": {
"jsonSchema": "string",
"datasource": "long"
},
"double": {
"jsonSchema": "string",
"datasource": "double"
},
"bigdecimal": {
"jsonSchema": "string",
"datasource": "double"
},
"integer": {
"jsonSchema": "string",
"datasource": "long"
}
}
},
"number": {
"arrival_format": [
"number",
"integer"
],
"store_format": {
"integer": {
"jsonSchema": "integer",
"datasource": "long"
},
"float": {
"jsonSchema": "number",
"datasource": "double"
},
"long": {
"jsonSchema": "integer",
"datasource": "long"
},
"double": {
"jsonSchema": "number",
"datasource": "double"
},
"bigdecimal": {
"jsonSchema": "number",
"datasource": "double"
},
"epoch": {
"jsonSchema": "integer",
"datasource": "long"
},
"number": {
"jsonSchema": "number",
"datasource": "double"
}
}
},
"object": {
"arrival_format": [
"object"
],
"store_format": {
"object": {
"jsonSchema": "object",
"datasource": "json"
}
}
},
"array": {
"arrival_format": [
"array"
],
"store_format": {
"array": {
"jsonSchema": "array",
"datasource": "array"
}
}
},
"boolean": {
"arrival_format": [
"boolean"
],
"store_format": {
"boolean": {
"jsonSchema": "boolean",
"datasource": "boolean"
}
}
}
},
"configurations": {
"indexConfiguration": {
"index": {
"Event Arrival Time": "obsrv_meta.syncts"
},
"rollupSuggestions": {
"summary": {
"tripID": {
"path": "$.tripID",
"cardinality": 99,
"index": false
}
}
}
},
"processing": {
"dedupKeys": [
"tripID"
],
"dropDuplicates": [
"Yes",
"No"
]
}
},
"mergedEvent": {
"tripID": "02e07922-e8a5-4655-84a8-b5ba1866f9fe",
"VendorID": "2",
"tpep_pickup_datetime": "2023-04-28 00:18:42",
"tpep_dropoff_datetime": "2024-02-15 00:24:38",
"passenger_count": "1",
"trip_distance": "1.60",
"RatecodeID": "1",
"store_and_fwd_flag": "N",
"PULocationID": "236",
"DOLocationID": "239",
"payment_type": "2",
"primary_passenger": {
"email": "Dewayne_Kuvalis17@gmail.com",
"mobile": "1-429-628-3797 x14211"
},
"fare_details": {
"fare_amount": "7",
"extra": "0.5",
"mta_tax": "0.5",
"tip_amount": "0",
"tolls_amount": "0",
"improvement_surcharge": "0.3",
"total_amount": "8.3",
"congestion_surcharge": ""
},
"passenger-name": "yashashk"
}
}
},
{
"dataset_id": "beckn-test-data",
"name": "beckn-test-data",
"type": "event",
"status": "ReadyToPublish",
"tags": [],
"version": 1,
"api_version": "v2",
"dataset_config": {
"data_key": "",
"timestamp_key": "ets",
"entry_topic": "beckn-test-data",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0,
"file_upload_path": [],
"exclude_fields": []
}
},
{
"dataset_id": "trip-details",
"name": "trip-details",
"type": "event",
"status": "ReadyToPublish",
"tags": [],
"version": 1,
"api_version": "v2",
"dataset_config": {
"data_key": "",
"timestamp_key": "obsrv_meta.syncts",
"entry_topic": "local.ingest",
"redis_db_host": "localhost",
"redis_db_port": 6379,
"index_data": true,
"redis_db": 0,
"file_upload_path": [],
"dataMappings": {
"text": {
"arrival_format": [
"string"
],
"store_format": {
"string": {
"jsonSchema": "string",
"datasource": "string"
},
"date-time": {
"jsonSchema": "string",
"datasource": "string"
},
"date": {
"jsonSchema": "string",
"datasource": "string"
},
"boolean": {
"jsonSchema": "string",
"datasource": "boolean"
},
"epoch": {
"jsonSchema": "string",
"datasource": "integer"
},
"long": {
"jsonSchema": "string",
"datasource": "long"
},
"double": {
"jsonSchema": "string",
"datasource": "double"
},
"bigdecimal": {
"jsonSchema": "string",
"datasource": "double"
},
"integer": {
"jsonSchema": "string",
"datasource": "long"
}
}
},
"number": {
"arrival_format": [
"number",
"integer"
],
"store_format": {
"integer": {
"jsonSchema": "integer",
"datasource": "long"
},
"float": {
"jsonSchema": "number",
"datasource": "double"
},
"long": {
"jsonSchema": "integer",
"datasource": "long"
},
"double": {
"jsonSchema": "number",
"datasource": "double"
},
"bigdecimal": {
"jsonSchema": "number",
"datasource": "double"
},
"epoch": {
"jsonSchema": "integer",
"datasource": "long"
},
"number": {
"jsonSchema": "number",
"datasource": "double"
}
}
},
"object": {
"arrival_format": [
"object"
],
"store_format": {
"object": {
"jsonSchema": "object",
"datasource": "json"
}
}
},
"array": {
"arrival_format": [
"array"
],
"store_format": {
"array": {
"jsonSchema": "array",
"datasource": "array"
}
}
},
"boolean": {
"arrival_format": [
"boolean"
],
"store_format": {
"boolean": {
"jsonSchema": "boolean",
"datasource": "boolean"
}
}
}
},
"configurations": {
"indexConfiguration": {
"index": {
"Event Arrival Time": "obsrv_meta.syncts"
},
"rollupSuggestions": {
"summary": {
"tripID": {
"path": "$.tripID",
"cardinality": 99,
"index": false
}
}
}
},
"processing": {
"dedupKeys": [
"tripID"
],
"dropDuplicates": [
"Yes",
"No"
]
}
},
"mergedEvent": {
"tripID": "02e07922-e8a5-4655-84a8-b5ba1866f9fe",
"VendorID": "2",
"tpep_pickup_datetime": "2023-04-28 00:18:42",
"tpep_dropoff_datetime": "2024-02-15 00:24:38",
"passenger_count": "1",
"trip_distance": "1.60",
"RatecodeID": "1",
"store_and_fwd_flag": "N",
"PULocationID": "236",
"DOLocationID": "239",
"payment_type": "2",
"primary_passenger": {
"email": "Dewayne_Kuvalis17@gmail.com",
"mobile": "1-429-628-3797 x14211"
},
"fare_details": {
"fare_amount": "7",
"extra": "0.5",
"mta_tax": "0.5",
"tip_amount": "0",
"tolls_amount": "0",
"improvement_surcharge": "0.3",
"total_amount": "8.3",
"congestion_surcharge": ""
},
"passenger-name": "yashashk"
}
}
}
],
"count": 30
}
}OK
This API generates presigned URLs to upload or download files from cloud
application/json{"id":"api.files.generate-url","ver":"v2","ts":"2024-04-19T12:58:47+05:30","params":{"msgid":"4a7f14c3-d61e-4d4f-be78-181834eeff6"},"request":{"files":["telemetry.json","school_data.json"],"access":"write"}}curl -L \
--request POST \
--url 'http://localhost:3000/v2/files/generate-url' \
--header 'Content-Type: application/json' \
--data '{
"id": "api.files.generate-url",
"ver": "v2",
"ts": "2024-04-19T12:58:47+05:30",
"params": {
"msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6"
},
"request": {
"files": [
"telemetry.json",
"school_data.json"
],
"access": "write"
}
}'{
"id": "api.files.generate-url",
"ver": "v2",
"ts": "2024-07-16T08:26:19+05:30",
"params": {
"status": "SUCCESS",
"msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6",
"resmsgid": "5306f309-4a15-458e-89e2-29d8ac0835d4"
},
"responseCode": "OK",
"result": [
{
"filePath": "test-connector/api-service/user_uploads/telemetry_10d595.json",
"fileName": "telemetry.json",
"preSignedUrl": "https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject"
},
{
"filePath": "test-connector/api-service/user_uploads/school_data_33109a.json",
"fileName": "school_data.json",
"preSignedUrl": "https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject"
}
]
}OK
This API allows you to perform status transition between 2 states. Allowed status transition are Draft to ReadyToPublish, ReadyToPublish to Live, Live to Retired and even Delete a dataset.
{"id":"api.datasets.status-transition","ver":"v2","ts":"2024-04-19T12:58:47+05:30","params":{"msgid":"4a7f14c3-d61e-4d4f-be78-181834eeff6"},"request":{"dataset_id":"telemetry-events","status":"ReadyToPublish"}}curl -L \
--request POST \
--url 'http://localhost:3000/v2/datasets/status-transition' \
--header 'Content-Type: application/json' \
--data '{
"id": "api.datasets.status-transition",
"ver": "v2",
"ts": "2024-04-19T12:58:47+05:30",
"params": {
"msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6"
},
"request": {
"dataset_id": "telemetry-events",
"status": "ReadyToPublish"
}
}'{
"id": "api.datasets.status-transition",
"ver": "v2",
"ts": "2024-05-30T12:18:54+05:30",
"params": {
"status": "SUCCESS",
"msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6",
"resmsgid": "5948e784-37f9-4a70-85ca-86c9077ee30b"
},
"responseCode": "OK",
"result": {
"message": "Dataset status transition to Delete successful",
"dataset_id": "trip-data"
}
}OK
This api is used to generate data schema for the given dataset event.
application/jsonconnect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA{"id":"api.datasets.dataschema","ver":"v2","ts":"2024-04-10T16:10:50+05:30","params":{"msgid":"4a7f14c3-d61e-4d4f-be78-181834eeff6d"},"request":{"data":[{"eid":"IMPRESSION","ets":1672657002221,"ver":"3.0","mid":"IMPRESSION:2b5834e196f485c17c4e49d292af43c0","actor":{"id":"0c45959486f579c24854d40a225d6161","type":"User"},"context":{"channel":"01268904781886259221","pdata":{"id":"staging.diksha.portal","ver":"5.1.0","pid":"sunbird-portal"},"env":"public","sid":"23850c90-8a8c-11ed-95d0-276800e1048c","did":"0c45959486f579c24854d40a225d6161","cdata":[],"rollup":{"l1":"01268904781886259221"},"uid":"anonymous"},"object":{},"tags":["01268904781886259221"],"edata":{"type":"view","pageid":"login","subtype":"pageexit","uri":"https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id=portal&state=254efd70-6b89-4f7d-868b-5c957f54174e&redirect_uri=https%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1&scope=openid&response_type=code&version=4","visits":[]},"syncts":1672657005814,"@timestamp":"2023-01-02T10:56:45.814Z","flags":{"ex_processed":true}},{"eid":"IMPRESSION","ets":1672656997928,"ver":"3.0","mid":"50263f0f-c2d5-4b15-95f4-5384c537f6cc","actor":{"id":"internal","type":"Consumer"},"context":{"channel":"0126796199493140480","pdata":{"id":"staging.sunbird.learning.service","pid":"learner-service","ver":"5.0.0"},"env":"Organisation","cdata":[{"id":"50263f0f-c2d5-4b15-95f4-5384c537f6cc","type":"Request"}],"rollup":{}},"edata":{"level":"info","type":"Api_access","message":"","params":[{"method":"POST"},{"url":"/v1/org/search"},{"duration":0},{"status":"OK"}]}},{"eid":"LOG","ets":1672656998024,"ver":"3.0","mid":"4a340ad0-0665-49b6-a1fa-a581dcac4550","actor":{"id":"internal","type":"Consumer"},"context":{"channel":"0126796199493140480","pdata":{"id":"staging.sunbird.learning.service","pid":"learner-service","ver":"5.0.0"},"env":"Organisation","cdata":[{"id":"4a340ad0-0665-49b6-a1fa-a581dcac4550","type":"Request"}],"rollup":{}},"edata":{"level":"info","type":"Api_access","message":"{eid='LOG', edata={level=trace, requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, type=system, message=EXIT LOG: method : POST, url: /v1/org/search , For Operation : orgSearch, params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, errmsg=Invalid value null for parameter hashTagId. Please provide a valid value., resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, err=UOS_ORGSER0017, status=FAILED, responseCode=400}]}}","params":[]}},{"eid":"LOG","ets":1672657004961,"ver":"3.0","mid":"f34112c7242a3e3a26f0015796b029c2","actor":{"id":"internal","type":"Consumer"},"context":{"channel":"0126796199493140480","pdata":{"id":"staging.sunbird.learning.service","pid":"learner-service","ver":"5.0.0"},"env":"Organisation","cdata":[{"id":"f34112c7242a3e3a26f0015796b029c2","type":"Request"}],"rollup":{}},"edata":{"level":"info","type":"Api_access","message":"ElasticSearchRestHighImpl:search: calling search for index org_alias, with query = {\"from\":0,\"size\":250,\"query\":{\"bool\":{\"must\":[{\"term\":{\"isTenant.raw\":{\"value\":true,\"boost\":1.0}}},{\"term\":{\"slug.raw\":{\"value\":\"ntp\",\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[],\"excludes\":[]}}","params":[]}},{"eid":"LOG","ets":1672657006595,"ver":"3.0","mid":"d23ff123-40f0-4262-a69b-b75b46d315a1","actor":{"id":"930a3994-cbe7-4e84-936f-4974096af6f2","type":"Consumer"},"context":{"channel":"0126796199493140480","pdata":{"id":"staging.sunbird.learning.service","pid":"learner-service","ver":"5.0.0"},"env":"User","cdata":[{"id":"d23ff123-40f0-4262-a69b-b75b46d315a1","type":"Request"}],"rollup":{}},"edata":{"level":"info","type":"Api_access","message":"{eid='LOG', edata={level=trace, requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, type=system, message=ENTRY LOG: method : GET, url: /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserRolesById, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}","params":[]}},{"eid":"LOG","ets":1672657006611,"ver":"3.0","mid":"7d944b1c-a906-4082-b42a-905aa6b78a4e","actor":{"id":"6ab35eea-01fd-4de0-8902-f68722caf859","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"staging.sunbird.learning.service","pid":"learner-service","ver":"5.0.0"},"env":"User","cdata":[{"id":"7d944b1c-a906-4082-b42a-905aa6b78a4e","type":"Request"}],"rollup":{}},"edata":{"level":"info","type":"Api_access","message":"{eid='LOG', edata={level=trace, requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, type=system, message=ENTRY LOG: method : GET, url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserProfileV5, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}","params":[]}},{"eid":"LOG","ets":1672657006620,"ver":"3.0","mid":"7d944b1c-a906-4082-b42a-905aa6b78a4e","actor":{"id":"6ab35eea-01fd-4de0-8902-f68722caf859","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"staging.sunbird.learning.service","pid":"learner-service","ver":"5.0.0"},"env":"User","cdata":[{"id":"7d944b1c-a906-4082-b42a-905aa6b78a4e","type":"Request"}],"rollup":{}},"edata":{"level":"info","type":"Api_access","message":"Cassandra query : SELECT * FROM sunbird.user_roles WHERE userId=?;","params":[]}},{"eid":"LOG","ets":1672657006645,"ver":"3.0","mid":"7d944b1c-a906-4082-b42a-905aa6b78a4e","actor":{"id":"6ab35eea-01fd-4de0-8902-f68722caf859","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"staging.sunbird.learning.service","pid":"learner-service","ver":"5.0.0"},"env":"User","cdata":[{"id":"7d944b1c-a906-4082-b42a-905aa6b78a4e","type":"Request"}],"rollup":{}},"edata":{"level":"info","type":"Api_access","message":"","params":[{"method":"GET"},{"url":"/v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859"},{"duration":0},{"status":"OK"}]}},{"eid":"LOG","ets":1672657007238,"ver":"3.0","mid":"d4d34fde-c407-efb6-03bd-9f892ca0f114","actor":{"id":"6ab35eea-01fd-4de0-8902-f68722caf859","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"staging.sunbird.portal","pid":"learner-service","ver":"5.0.0"},"env":"User","did":"d904c90d9f81ddac20141b94ddd606a0","cdata":[{"id":"d4d34fde-c407-efb6-03bd-9f892ca0f114","type":"Request"}],"rollup":{}},"edata":{"level":"info","type":"Api_access","message":"Cassandra query : SELECT * FROM sunbird.user WHERE id=?;","params":[]}}],"config":{"dataset":"financial_transactions"}}}curl -L \
--request POST \
--url 'http://localhost:3000/v2/datasets/dataschema' \
--header 'Content-Type: application/json' \
--data '{
"id": "api.datasets.dataschema",
"ver": "v2",
"ts": "2024-04-10T16:10:50+05:30",
"params": {
"msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d"
},
"request": {
"data": [
{
"eid": "IMPRESSION",
"ets": 1672657002221,
"ver": "3.0",
"mid": "IMPRESSION:2b5834e196f485c17c4e49d292af43c0",
"actor": {
"id": "0c45959486f579c24854d40a225d6161",
"type": "User"
},
"context": {
"channel": "01268904781886259221",
"pdata": {
"id": "staging.diksha.portal",
"ver": "5.1.0",
"pid": "sunbird-portal"
},
"env": "public",
"sid": "23850c90-8a8c-11ed-95d0-276800e1048c",
"did": "0c45959486f579c24854d40a225d6161",
"cdata": [],
"rollup": {
"l1": "01268904781886259221"
},
"uid": "anonymous"
},
"object": {},
"tags": [
"01268904781886259221"
],
"edata": {
"type": "view",
"pageid": "login",
"subtype": "pageexit",
"uri": "https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id=portal&state=254efd70-6b89-4f7d-868b-5c957f54174e&redirect_uri=https%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1&scope=openid&response_type=code&version=4",
"visits": []
},
"syncts": 1672657005814,
"@timestamp": "2023-01-02T10:56:45.814Z",
"flags": {
"ex_processed": true
}
},
{
"eid": "IMPRESSION",
"ets": 1672656997928,
"ver": "3.0",
"mid": "50263f0f-c2d5-4b15-95f4-5384c537f6cc",
"actor": {
"id": "internal",
"type": "Consumer"
},
"context": {
"channel": "0126796199493140480",
"pdata": {
"id": "staging.sunbird.learning.service",
"pid": "learner-service",
"ver": "5.0.0"
},
"env": "Organisation",
"cdata": [
{
"id": "50263f0f-c2d5-4b15-95f4-5384c537f6cc",
"type": "Request"
}
],
"rollup": {}
},
"edata": {
"level": "info",
"type": "Api_access",
"message": "",
"params": [
{
"method": "POST"
},
{
"url": "/v1/org/search"
},
{
"duration": 0
},
{
"status": "OK"
}
]
}
},
{
"eid": "LOG",
"ets": 1672656998024,
"ver": "3.0",
"mid": "4a340ad0-0665-49b6-a1fa-a581dcac4550",
"actor": {
"id": "internal",
"type": "Consumer"
},
"context": {
"channel": "0126796199493140480",
"pdata": {
"id": "staging.sunbird.learning.service",
"pid": "learner-service",
"ver": "5.0.0"
},
"env": "Organisation",
"cdata": [
{
"id": "4a340ad0-0665-49b6-a1fa-a581dcac4550",
"type": "Request"
}
],
"rollup": {}
},
"edata": {
"level": "info",
"type": "Api_access",
"message": "{eid='LOG', edata={level=trace, requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, type=system, message=EXIT LOG: method : POST, url: /v1/org/search , For Operation : orgSearch, params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, errmsg=Invalid value null for parameter hashTagId. Please provide a valid value., resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, err=UOS_ORGSER0017, status=FAILED, responseCode=400}]}}",
"params": []
}
},
{
"eid": "LOG",
"ets": 1672657004961,
"ver": "3.0",
"mid": "f34112c7242a3e3a26f0015796b029c2",
"actor": {
"id": "internal",
"type": "Consumer"
},
"context": {
"channel": "0126796199493140480",
"pdata": {
"id": "staging.sunbird.learning.service",
"pid": "learner-service",
"ver": "5.0.0"
},
"env": "Organisation",
"cdata": [
{
"id": "f34112c7242a3e3a26f0015796b029c2",
"type": "Request"
}
],
"rollup": {}
},
"edata": {
"level": "info",
"type": "Api_access",
"message": "ElasticSearchRestHighImpl:search: calling search for index org_alias, with query = {\"from\":0,\"size\":250,\"query\":{\"bool\":{\"must\":[{\"term\":{\"isTenant.raw\":{\"value\":true,\"boost\":1.0}}},{\"term\":{\"slug.raw\":{\"value\":\"ntp\",\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[],\"excludes\":[]}}",
"params": []
}
},
{
"eid": "LOG",
"ets": 1672657006595,
"ver": "3.0",
"mid": "d23ff123-40f0-4262-a69b-b75b46d315a1",
"actor": {
"id": "930a3994-cbe7-4e84-936f-4974096af6f2",
"type": "Consumer"
},
"context": {
"channel": "0126796199493140480",
"pdata": {
"id": "staging.sunbird.learning.service",
"pid": "learner-service",
"ver": "5.0.0"
},
"env": "User",
"cdata": [
{
"id": "d23ff123-40f0-4262-a69b-b75b46d315a1",
"type": "Request"
}
],
"rollup": {}
},
"edata": {
"level": "info",
"type": "Api_access",
"message": "{eid='LOG', edata={level=trace, requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, type=system, message=ENTRY LOG: method : GET, url: /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserRolesById, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}",
"params": []
}
},
{
"eid": "LOG",
"ets": 1672657006611,
"ver": "3.0",
"mid": "7d944b1c-a906-4082-b42a-905aa6b78a4e",
"actor": {
"id": "6ab35eea-01fd-4de0-8902-f68722caf859",
"type": "User"
},
"context": {
"channel": "0126796199493140480",
"pdata": {
"id": "staging.sunbird.learning.service",
"pid": "learner-service",
"ver": "5.0.0"
},
"env": "User",
"cdata": [
{
"id": "7d944b1c-a906-4082-b42a-905aa6b78a4e",
"type": "Request"
}
],
"rollup": {}
},
"edata": {
"level": "info",
"type": "Api_access",
"message": "{eid='LOG', edata={level=trace, requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, type=system, message=ENTRY LOG: method : GET, url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserProfileV5, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}",
"params": []
}
},
{
"eid": "LOG",
"ets": 1672657006620,
"ver": "3.0",
"mid": "7d944b1c-a906-4082-b42a-905aa6b78a4e",
"actor": {
"id": "6ab35eea-01fd-4de0-8902-f68722caf859",
"type": "User"
},
"context": {
"channel": "0126796199493140480",
"pdata": {
"id": "staging.sunbird.learning.service",
"pid": "learner-service",
"ver": "5.0.0"
},
"env": "User",
"cdata": [
{
"id": "7d944b1c-a906-4082-b42a-905aa6b78a4e",
"type": "Request"
}
],
"rollup": {}
},
"edata": {
"level": "info",
"type": "Api_access",
"message": "Cassandra query : SELECT * FROM sunbird.user_roles WHERE userId=?;",
"params": []
}
},
{
"eid": "LOG",
"ets": 1672657006645,
"ver": "3.0",
"mid": "7d944b1c-a906-4082-b42a-905aa6b78a4e",
"actor": {
"id": "6ab35eea-01fd-4de0-8902-f68722caf859",
"type": "User"
},
"context": {
"channel": "0126796199493140480",
"pdata": {
"id": "staging.sunbird.learning.service",
"pid": "learner-service",
"ver": "5.0.0"
},
"env": "User",
"cdata": [
{
"id": "7d944b1c-a906-4082-b42a-905aa6b78a4e",
"type": "Request"
}
],
"rollup": {}
},
"edata": {
"level": "info",
"type": "Api_access",
"message": "",
"params": [
{
"method": "GET"
},
{
"url": "/v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859"
},
{
"duration": 0
},
{
"status": "OK"
}
]
}
},
{
"eid": "LOG",
"ets": 1672657007238,
"ver": "3.0",
"mid": "d4d34fde-c407-efb6-03bd-9f892ca0f114",
"actor": {
"id": "6ab35eea-01fd-4de0-8902-f68722caf859",
"type": "User"
},
"context": {
"channel": "0126796199493140480",
"pdata": {
"id": "staging.sunbird.portal",
"pid": "learner-service",
"ver": "5.0.0"
},
"env": "User",
"did": "d904c90d9f81ddac20141b94ddd606a0",
"cdata": [
{
"id": "d4d34fde-c407-efb6-03bd-9f892ca0f114",
"type": "Request"
}
],
"rollup": {}
},
"edata": {
"level": "info",
"type": "Api_access",
"message": "Cassandra query : SELECT * FROM sunbird.user WHERE id=?;",
"params": []
}
}
],
"config": {
"dataset": "financial_transactions"
}
}
}'{
"id": "api.datasets.dataschema",
"ver": "v2",
"ts": "2024-07-22T12:32:50+05:30",
"params": {
"status": "SUCCESS",
"msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d",
"resmsgid": "1309aea0-9a97-46e9-bc5e-a16a8a7fb624"
},
"responseCode": "OK",
"result": {
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"eid": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"ets": {
"type": "integer",
"suggestions": [
{
"message": "The Property 'ets' appears to be 'epoch' format type.",
"severity": "",
"path": "properties.ets"
}
],
"arrival_format": "number",
"data_type": "epoch"
},
"ver": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"mid": {
"type": "string",
"suggestions": [
{
"message": "The Property 'mid' appears to be 'uuid' format type.",
"advice": "Suggest to not to index the high cardinal columns",
"resolutionType": "DEDUP",
"severity": "LOW",
"path": "properties.mid"
}
],
"arrival_format": "text",
"data_type": "string"
},
"actor": {
"type": "object",
"properties": {
"id": {
"type": "string",
"suggestions": [
{
"message": "The Property 'actor.id' appears to be 'uuid' format type.",
"advice": "Suggest to not to index the high cardinal columns",
"resolutionType": "DEDUP",
"severity": "LOW",
"path": "properties.actor.properties.id"
}
],
"arrival_format": "text",
"data_type": "string"
},
"type": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
}
},
"arrival_format": "object",
"data_type": "object"
},
"context": {
"type": "object",
"properties": {
"channel": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"pdata": {
"type": "object",
"properties": {
"id": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"ver": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"pid": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
}
},
"arrival_format": "object",
"data_type": "object"
},
"env": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"sid": {
"type": "string",
"suggestions": [
{
"message": "Conflict in the Schema Generation at property: 'sid'. The property sid: only 1 time(s) appeared ",
"advice": "The Property looks to be Optional. System has updated the property schema to optional",
"resolutionType": "OPTIONAL",
"severity": "MEDIUM",
"path": "properties.context.properties.sid"
},
{
"message": "The Property 'context.sid' appears to be 'uuid' format type.",
"advice": "Suggest to not to index the high cardinal columns",
"resolutionType": "DEDUP",
"severity": "LOW",
"path": "properties.context.properties.sid"
}
],
"arrival_format": "text",
"data_type": "string"
},
"did": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"cdata": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"type": "string",
"suggestions": [
{
"message": "The Property 'context.cdata[*].id' appears to be 'uuid' format type.",
"advice": "Suggest to not to index the high cardinal columns",
"resolutionType": "DEDUP",
"severity": "LOW",
"path": "properties.context.properties.cdata.items.properties.id"
}
],
"arrival_format": "text",
"data_type": "string"
},
"type": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
}
}
},
"arrival_format": "array",
"data_type": "array"
},
"rollup": {
"type": "object",
"properties": {
"l1": {
"type": "string",
"suggestions": [
{
"message": "Conflict in the Schema Generation at property: 'l1'. The property l1: only 1 time(s) appeared ",
"advice": "The Property looks to be Optional. System has updated the property schema to optional",
"resolutionType": "OPTIONAL",
"severity": "MEDIUM",
"path": "properties.context.properties.rollup.properties.l1"
}
],
"arrival_format": "text",
"data_type": "string"
}
},
"arrival_format": "object",
"data_type": "object"
},
"uid": {
"type": "string",
"suggestions": [
{
"message": "Conflict in the Schema Generation at property: 'uid'. The property uid: only 1 time(s) appeared ",
"advice": "The Property looks to be Optional. System has updated the property schema to optional",
"resolutionType": "OPTIONAL",
"severity": "MEDIUM",
"path": "properties.context.properties.uid"
}
],
"arrival_format": "text",
"data_type": "string"
}
},
"arrival_format": "object",
"data_type": "object"
},
"object": {
"type": "object",
"suggestions": [
{
"message": "Conflict in the Schema Generation at property: 'object'. The property object: only 1 time(s) appeared ",
"advice": "The Property looks to be Optional. System has updated the property schema to optional",
"resolutionType": "OPTIONAL",
"severity": "MEDIUM",
"path": "properties.object"
}
],
"arrival_format": "object",
"data_type": "object"
},
"tags": {
"type": "array",
"items": {
"type": "string"
},
"suggestions": [
{
"message": "Conflict in the Schema Generation at property: 'tags'. The property tags: only 1 time(s) appeared ",
"advice": "The Property looks to be Optional. System has updated the property schema to optional",
"resolutionType": "OPTIONAL",
"severity": "MEDIUM",
"path": "properties.tags"
}
],
"arrival_format": "array",
"data_type": "array"
},
"edata": {
"type": "object",
"properties": {
"type": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"pageid": {
"type": "string",
"suggestions": [
{
"message": "Conflict in the Schema Generation at property: 'pageid'. The property pageid: only 1 time(s) appeared ",
"advice": "The Property looks to be Optional. System has updated the property schema to optional",
"resolutionType": "OPTIONAL",
"severity": "MEDIUM",
"path": "properties.edata.properties.pageid"
}
],
"arrival_format": "text",
"data_type": "string"
},
"subtype": {
"type": "string",
"suggestions": [
{
"message": "Conflict in the Schema Generation at property: 'subtype'. The property subtype: only 1 time(s) appeared ",
"advice": "The Property looks to be Optional. System has updated the property schema to optional",
"resolutionType": "OPTIONAL",
"severity": "MEDIUM",
"path": "properties.edata.properties.subtype"
}
],
"arrival_format": "text",
"data_type": "string"
},
"uri": {
"type": "string",
"suggestions": [
{
"message": "Conflict in the Schema Generation at property: 'uri'. The property uri: only 1 time(s) appeared ",
"advice": "The Property looks to be Optional. System has updated the property schema to optional",
"resolutionType": "OPTIONAL",
"severity": "MEDIUM",
"path": "properties.edata.properties.uri"
},
{
"message": "The Property 'edata.uri' appears to be 'uri' format type.",
"severity": "",
"path": "properties.edata.properties.uri"
}
],
"arrival_format": "text",
"data_type": "string"
},
"visits": {
"type": "array",
"items": {
"type": "object",
"properties": {
"key": {
"type": "string"
},
"value": {
"type": "string"
}
}
},
"suggestions": [
{
"message": "Conflict in the Schema Generation at property: 'visits'. The property visits: only 1 time(s) appeared ",
"advice": "The Property looks to be Optional. System has updated the property schema to optional",
"resolutionType": "OPTIONAL",
"severity": "MEDIUM",
"path": "properties.edata.properties.visits"
}
],
"arrival_format": "array",
"data_type": "array"
},
"level": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"message": {
"type": "string",
"arrival_format": "text",
"data_type": "string"
},
"params": {
"type": "array",
"items": {
"type": "object",
"properties": {
"key": {
"type": "string"
},
"value": {
"type": "string"
}
}
},
"arrival_format": "array",
"data_type": "array"
}
},
"arrival_format": "object",
"data_type": "object"
},
"syncts": {
"type": "integer",
"suggestions": [
{
"message": "Conflict in the Schema Generation at property: 'syncts'. The property syncts: only 1 time(s) appeared ",
"advice": "The Property looks to be Optional. System has updated the property schema to optional",
"resolutionType": "OPTIONAL",
"severity": "MEDIUM",
"path": "properties.syncts"
},
{
"message": "The Property 'syncts' appears to be 'epoch' format type.",
"severity": "",
"path": "properties.syncts"
}
],
"arrival_format": "number",
"data_type": "epoch"
},
"@timestamp": {
"type": "string",
"suggestions": [
{
"message": "Conflict in the Schema Generation at property: '@timestamp'. The property @timestamp: only 1 time(s) appeared ",
"advice": "The Property looks to be Optional. System has updated the property schema to optional",
"resolutionType": "OPTIONAL",
"severity": "MEDIUM",
"path": "properties.@timestamp"
},
{
"message": "The Property '@timestamp' appears to be 'date-time' format type.",
"advice": "The System can index all data on this column",
"resolutionType": "INDEX",
"severity": "LOW",
"path": "properties.@timestamp"
}
],
"arrival_format": "text",
"data_type": "date-time"
},
"flags": {
"type": "object",
"properties": {
"ex_processed": {
"type": "boolean",
"suggestions": [
{
"message": "Conflict in the Schema Generation at property: 'ex_processed'. The property ex_processed: only 1 time(s) appeared ",
"advice": "The Property looks to be Optional. System has updated the property schema to optional",
"resolutionType": "OPTIONAL",
"severity": "MEDIUM",
"path": "properties.flags.properties.ex_processed"
}
],
"arrival_format": "boolean",
"data_type": "boolean"
}
},
"suggestions": [
{
"message": "Conflict in the Schema Generation at property: 'flags'. The property flags: only 1 time(s) appeared ",
"advice": "The Property looks to be Optional. System has updated the property schema to optional",
"resolutionType": "OPTIONAL",
"severity": "MEDIUM",
"path": "properties.flags"
}
],
"arrival_format": "object",
"data_type": "object"
}
},
"additionalProperties": true
},
"configurations": {
"indexConfiguration": {
"index": {
"Event Arrival Time": "obsrv_meta.syncts"
},
"rollupSuggestions": {
"summary": {
"mid": {
"path": "$.mid",
"cardinality": 67,
"index": false
},
"actor.id": {
"path": "$.actor.properties.id",
"cardinality": 56,
"index": false
},
"context.sid": {
"path": "$.context.properties.sid",
"cardinality": 11,
"index": true
},
"edata.uri": {
"path": "$.edata.properties.uri",
"cardinality": 11,
"index": true
},
"context.cdata[*].id": {
"path": "$.context.properties.cdata.items.properties.id",
"cardinality": 62,
"index": false
}
}
}
},
"processing": {
"dedupKeys": [
"mid",
"context.cdata[*].id",
"actor.id"
],
"dropDuplicates": [
"Yes",
"No"
]
}
},
"dataMappings": {
"text": {
"arrival_format": [
"string"
],
"store_format": {
"string": {
"jsonSchema": "string",
"datasource": "string"
},
"date-time": {
"jsonSchema": "string",
"datasource": "string"
},
"date": {
"jsonSchema": "string",
"datasource": "string"
},
"boolean": {
"jsonSchema": "string",
"datasource": "boolean"
},
"epoch": {
"jsonSchema": "string",
"datasource": "integer"
},
"long": {
"jsonSchema": "string",
"datasource": "long"
},
"double": {
"jsonSchema": "string",
"datasource": "double"
},
"bigdecimal": {
"jsonSchema": "string",
"datasource": "double"
},
"integer": {
"jsonSchema": "string",
"datasource": "long"
}
}
},
"number": {
"arrival_format": [
"number",
"integer"
],
"store_format": {
"integer": {
"jsonSchema": "integer",
"datasource": "long"
},
"float": {
"jsonSchema": "number",
"datasource": "double"
},
"long": {
"jsonSchema": "integer",
"datasource": "long"
},
"double": {
"jsonSchema": "number",
"datasource": "double"
},
"bigdecimal": {
"jsonSchema": "number",
"datasource": "double"
},
"epoch": {
"jsonSchema": "integer",
"datasource": "long"
},
"number": {
"jsonSchema": "number",
"datasource": "double"
}
}
},
"object": {
"arrival_format": [
"object"
],
"store_format": {
"object": {
"jsonSchema": "object",
"datasource": "json"
}
}
},
"array": {
"arrival_format": [
"array"
],
"store_format": {
"array": {
"jsonSchema": "array",
"datasource": "array"
}
}
},
"boolean": {
"arrival_format": [
"boolean"
],
"store_format": {
"boolean": {
"jsonSchema": "boolean",
"datasource": "boolean"
}
}
}
}
}
}OK
This API allows you to perform status transition between 2 states. Allowed status transition are Draft to ReadyToPublish, ReadyToPublish to Live, Live to Retired and even Delete a dataset.
{"id":"api.datasets.status-transition","ver":"v2","ts":"2024-04-19T12:58:47+05:30","params":{"msgid":"4a7f14c3-d61e-4d4f-be78-181834eeff6"},"request":{"dataset_id":"telemetry-events","status":"ReadyToPublish"}}curl -L \
--request POST \
--url 'http://localhost:3000/v2/datasets/status-transition' \
--header 'Content-Type: application/json' \
--data '{
"id": "api.datasets.status-transition",
"ver": "v2",
"ts": "2024-04-19T12:58:47+05:30",
"params": {
"msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6"
},
"request": {
"dataset_id": "telemetry-events",
"status": "ReadyToPublish"
}
}'{
"id": "api.datasets.status-transition",
"ver": "v2",
"ts": "2024-05-30T12:18:54+05:30",
"params": {
"status": "SUCCESS",
"msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6",
"resmsgid": "5948e784-37f9-4a70-85ca-86c9077ee30b"
},
"responseCode": "OK",
"result": {
"message": "Dataset status transition to Delete successful",
"dataset_id": "trip-data"
}
}OK
application/json{"id":"api.datasets.copy","ver":"v2","ts":"2024-05-21T14:30:00Z","params":{"msgid":"127384e4a-a051-4a9f-9b3f-a64a8034fad7"},"request":{"source":{"datasetId":"dataset-telemetry","isLive":true},"destination":{"datasetId":"bew-copy-live2"}}}curl -L \
--request POST \
--url 'http://localhost:3000/v2/datasets/copy' \
--header 'Content-Type: application/json' \
--data '{
"id": "api.datasets.copy",
"ver": "v2",
"ts": "2024-05-21T14:30:00Z",
"params": {
"msgid": "127384e4a-a051-4a9f-9b3f-a64a8034fad7"
},
"request": {
"source": {
"datasetId": "dataset-telemetry",
"isLive": true
},
"destination": {
"datasetId": "bew-copy-live2"
}
}
}'{
"id": "api.datasets.copy",
"ver": "v2",
"ts": "2024-11-19T12:56:12+05:30",
"params": {
"status": "SUCCESS",
"msgid": "127384e4a-a051-4a9f-9b3f-a64a8034fad7",
"resmsgid": "28072631-8e80-45eb-906f-d933a90646d0"
},
"responseCode": "OK",
"result": {
"dataset_id": "new-copy-live2",
"message": "Dataset clone successful"
}
}OK