From 835393a63185a3bc59cefcd8c68564786083e726 Mon Sep 17 00:00:00 2001 From: Sergey Chernetsky Date: Thu, 4 Aug 2022 18:16:56 +0800 Subject: [PATCH 1/4] Migration created and schema update --- .../docker-compose.hasura.yml | 20 ++++++++++++++++ hasura/hasura_metadata.json | 2 +- ...-token-properties-and-attributes-fields.js | 23 +++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 docker/polkastats-backend/docker-compose.hasura.yml create mode 100644 sequelize/migrations/20220804094054-token-properties-and-attributes-fields.js diff --git a/docker/polkastats-backend/docker-compose.hasura.yml b/docker/polkastats-backend/docker-compose.hasura.yml new file mode 100644 index 00000000..bb10c6d4 --- /dev/null +++ b/docker/polkastats-backend/docker-compose.hasura.yml @@ -0,0 +1,20 @@ +version: '3.7' + +services: + # + # Hasura + # + graphql-engine: + image: hasura/graphql-engine:v2.2.0 + ports: + - '8080:8080' + environment: + HASURA_GRAPHQL_DATABASE_URL: '${GRAPHQL_DATABASE_URL}' # postgres://polkastats:polkastats@host.docker.internal:5432/polkastats + HASURA_GRAPHQL_ENABLE_CONSOLE: '${GRAPHQL_ENABLE_CONSOLE}' # set to "false" to disable console + HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log + HASURA_GRAPHQL_DISABLE_CORS: '${GRAPHQL_DISABLE_CORS}' + HASURA_GRAPHQL_UNAUTHORIZED_ROLE: 'anonymous' + HASURA_GRAPHQL_JWT_SECRET: '${GRAPHQL_JWT_SECRET}' + ## uncomment next line to set an admin secret + HASURA_GRAPHQL_ADMIN_SECRET: '${GRAPHQL_ADMIN_SECRET}' + HASURA_GRAPHQL_PG_CONNECTIONS: '${GRAPHQL_PG_CONNECTIONS}' diff --git a/hasura/hasura_metadata.json b/hasura/hasura_metadata.json index 8962ecf2..362b3f48 100644 --- a/hasura/hasura_metadata.json +++ b/hasura/hasura_metadata.json @@ -1 +1 @@ -{"version":3,"sources":[{"name":"default","kind":"postgres","tables":[{"table":{"schema":"public","name":"SequelizeMeta"}},{"table":{"schema":"public","name":"account"},"select_permissions":[{"role":"anonymous","permission":{"columns":["account_id","balances","available_balance","free_balance","locked_balance","nonce","timestamp","block_height","account_id_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"block"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_number","session_length","timestamp","need_rescan","new_accounts","num_transfers","spec_version","total_events","block_hash","extrinsics_root","parent_hash","spec_name","state_root","total_issuance"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"collections"},"array_relationships":[{"name":"tokens","using":{"manual_configuration":{"remote_table":{"schema":"public","name":"tokens"},"insertion_order":null,"column_mapping":{"collection_id":"collection_id"}}}}],"select_permissions":[{"role":"anonymous","permission":{"columns":["attributes_schema","collection_cover","collection_id","const_chain_schema","date_of_creation","description","limits_account_ownership","limits_sponsore_data_rate","limits_sponsore_data_size","mint_mode","mode","name","offchain_schema","owner","owner_can_destroy","owner_can_transfer","owner_normalized","properties","schema_version","sponsorship","token_limit","token_prefix","variable_on_chain_schema"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"event"}},{"table":{"schema":"public","name":"extrinsic"},"select_permissions":[{"role":"anonymous","permission":{"columns":["amount","args","block_index","block_number","extrinsic_index","fee","hash","is_signed","method","section","signer","signer_normalized","success","timestamp","to_owner","to_owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"system"}},{"table":{"schema":"public","name":"tokens"},"object_relationships":[{"name":"collection","using":{"manual_configuration":{"remote_table":{"schema":"public","name":"collections"},"insertion_order":null,"column_mapping":{"collection_id":"collection_id"}}}}],"select_permissions":[{"role":"anonymous","permission":{"columns":["date_of_creation","id","owner","token_id","data","owner_normalized","collection_id","parent_id"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"total"},"select_permissions":[{"role":"anonymous","permission":{"columns":["count","name"],"filter":{}}}]},{"table":{"schema":"public","name":"view_collections"},"select_permissions":[{"role":"anonymous","permission":{"columns":["actions_count","collection_cover","collection_id","const_chain_schema","date_of_creation","description","holders_count","limits_account_ownership","limits_sponsore_data_rate","limits_sponsore_data_size","mint_mode","name","offchain_schema","owner","owner_can_destroy","owner_can_transfer","owner_normalized","schema_version","sponsorship","token_limit","token_prefix","tokens_count","type"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_extrinsic"},"select_permissions":[{"role":"anonymous","permission":{"columns":["amount","block_index","block_number","fee","from_owner","from_owner_normalized","hash","method","section","success","timestamp","to_owner","to_owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_holders"},"select_permissions":[{"role":"anonymous","permission":{"columns":["collection_id","count","owner","owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_last_block"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_number","event_count","extrinsic_count","timestamp"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_tokens"},"select_permissions":[{"role":"anonymous","permission":{"columns":["collection_cover","collection_description","collection_id","collection_name","collection_owner","collection_owner_normalized","data","date_of_creation","image_path","is_sold","owner","owner_normalized","token_id","token_name","token_prefix"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_transfer"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_index","section","method","data"],"filter":{},"allow_aggregations":true}}]}],"configuration":{"connection_info":{"use_prepared_statements":true,"database_url":{"from_env":"HASURA_GRAPHQL_DATABASE_URL"},"isolation_level":"read-committed","pool_settings":{"connection_lifetime":600,"retries":1,"idle_timeout":180,"max_connections":2}}}}]} \ No newline at end of file +{"version":3,"sources":[{"name":"default","kind":"postgres","tables":[{"table":{"schema":"public","name":"SequelizeMeta"}},{"table":{"schema":"public","name":"account"},"select_permissions":[{"role":"anonymous","permission":{"columns":["account_id","balances","available_balance","free_balance","locked_balance","nonce","timestamp","block_height","account_id_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"block"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_number","session_length","timestamp","need_rescan","new_accounts","num_transfers","spec_version","total_events","block_hash","extrinsics_root","parent_hash","spec_name","state_root","total_issuance"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"collections"},"array_relationships":[{"name":"tokens","using":{"manual_configuration":{"remote_table":{"schema":"public","name":"tokens"},"insertion_order":null,"column_mapping":{"collection_id":"collection_id"}}}}],"select_permissions":[{"role":"anonymous","permission":{"columns":["attributes_schema","collection_cover","collection_id","const_chain_schema","date_of_creation","description","limits_account_ownership","limits_sponsore_data_rate","limits_sponsore_data_size","mint_mode","mode","name","offchain_schema","owner","owner_can_destroy","owner_can_transfer","owner_normalized","properties","schema_version","sponsorship","token_limit","token_prefix","variable_on_chain_schema"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"event"}},{"table":{"schema":"public","name":"extrinsic"},"select_permissions":[{"role":"anonymous","permission":{"columns":["amount","args","block_index","block_number","extrinsic_index","fee","hash","is_signed","method","section","signer","signer_normalized","success","timestamp","to_owner","to_owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"system"}},{"table":{"schema":"public","name":"tokens"},"object_relationships":[{"name":"collection","using":{"manual_configuration":{"remote_table":{"schema":"public","name":"collections"},"insertion_order":null,"column_mapping":{"collection_id":"collection_id"}}}}],"select_permissions":[{"role":"anonymous","permission":{"columns":["attributes","collection_id","data","date_of_creation","id","owner","owner_normalized","parent_id","properties","token_id"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"total"},"select_permissions":[{"role":"anonymous","permission":{"columns":["count","name"],"filter":{}}}]},{"table":{"schema":"public","name":"view_collections"},"select_permissions":[{"role":"anonymous","permission":{"columns":["actions_count","collection_cover","collection_id","const_chain_schema","date_of_creation","description","holders_count","limits_account_ownership","limits_sponsore_data_rate","limits_sponsore_data_size","mint_mode","name","offchain_schema","owner","owner_can_destroy","owner_can_transfer","owner_normalized","schema_version","sponsorship","token_limit","token_prefix","tokens_count","type"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_extrinsic"},"select_permissions":[{"role":"anonymous","permission":{"columns":["amount","block_index","block_number","fee","from_owner","from_owner_normalized","hash","method","section","success","timestamp","to_owner","to_owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_holders"},"select_permissions":[{"role":"anonymous","permission":{"columns":["collection_id","count","owner","owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_last_block"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_number","event_count","extrinsic_count","timestamp"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_tokens"},"select_permissions":[{"role":"anonymous","permission":{"columns":["collection_cover","collection_description","collection_id","collection_name","collection_owner","collection_owner_normalized","data","date_of_creation","image_path","is_sold","owner","owner_normalized","token_id","token_name","token_prefix"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_transfer"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_index","section","method","data"],"filter":{},"allow_aggregations":true}}]}],"configuration":{"connection_info":{"use_prepared_statements":true,"database_url":{"from_env":"HASURA_GRAPHQL_DATABASE_URL"},"isolation_level":"read-committed","pool_settings":{"connection_lifetime":600,"retries":1,"idle_timeout":180,"max_connections":2}}}}]} \ No newline at end of file diff --git a/sequelize/migrations/20220804094054-token-properties-and-attributes-fields.js b/sequelize/migrations/20220804094054-token-properties-and-attributes-fields.js new file mode 100644 index 00000000..019801a1 --- /dev/null +++ b/sequelize/migrations/20220804094054-token-properties-and-attributes-fields.js @@ -0,0 +1,23 @@ +'use strict'; + +module.exports = { + async up (queryInterface, Sequelize) { + return Promise.all([ + queryInterface.addColumn('tokens', 'properties', { + type: Sequelize.DataTypes.JSONB, + defaultValue: [] + }), + queryInterface.addColumn('tokens', 'attributes', { + type: Sequelize.DataTypes.JSONB, + defaultValue: {} + }), + ]) + }, + + async down (queryInterface, Sequelize) { + return Promise.all([ + await queryInterface.removeColumn('tokens', 'properties'), + await queryInterface.removeColumn('tokens', 'attributes'), + ]); + } +}; From 1108e3a7c09a32f81240ecc728e27b701dd04c47 Mon Sep 17 00:00:00 2001 From: Sergey Chernetsky Date: Fri, 5 Aug 2022 14:11:53 +0800 Subject: [PATCH 2/4] Tokens properties and attributes saving --- .../bridgeProviderAPI/concreate/opalAPI.ts | 23 +++- .../implement/implementOpalAPI.ts | 18 +-- lib/token/tokenData.ts | 127 ++++++------------ lib/token/tokenDb.ts | 2 + lib/token/tokenDbEntity.interface.ts | 3 + utils/utils.js | 13 +- 6 files changed, 85 insertions(+), 101 deletions(-) diff --git a/lib/providerAPI/bridgeProviderAPI/concreate/opalAPI.ts b/lib/providerAPI/bridgeProviderAPI/concreate/opalAPI.ts index d8096e05..30a012c5 100644 --- a/lib/providerAPI/bridgeProviderAPI/concreate/opalAPI.ts +++ b/lib/providerAPI/bridgeProviderAPI/concreate/opalAPI.ts @@ -1,5 +1,8 @@ -import { CollectionInfoWithSchema } from '@unique-nft/sdk/tokens'; -import { UpDataStructsCollectionLimits, UpDataStructsRpcCollection } from '@unique-nft/unique-mainnet-types'; +import { CollectionInfoWithSchema, TokenPropertiesResult, UniqueTokenDecoded } from '@unique-nft/sdk/tokens'; +import { + UpDataStructsCollectionLimits, + UpDataStructsRpcCollection, +} from '@unique-nft/unique-mainnet-types'; import { ImplementOpalAPI } from '../implement/implementOpalAPI'; import AbstractAPI from './abstractAPI'; @@ -26,9 +29,19 @@ export class OpalAPI extends AbstractAPI { }; } - async getToken(collectionId, tokenId) { - const token = await this.impl.impGetToken(collectionId, tokenId); - return token || null; + async getToken(collectionId, tokenId): Promise<{ + tokenDecoded: UniqueTokenDecoded | null, + tokenProperties: TokenPropertiesResult | null + }> { + const [tokenDecoded, tokenProperties] = await Promise.all([ + this.impl.impGetToken(collectionId, tokenId), + this.impl.impGetTokenPropertiesSdk(collectionId, tokenId) + ]); + + return { + tokenDecoded, + tokenProperties + }; } getCollectionCount() { diff --git a/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts b/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts index 97fa61ac..9b4b35d1 100644 --- a/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts +++ b/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts @@ -1,10 +1,9 @@ /* eslint-disable import/no-duplicates */ import { UpDataStructsCollectionLimits, - UpDataStructsRpcCollection, - UpDataStructsTokenData, + UpDataStructsRpcCollection } from '@unique-nft/unique-mainnet-types'; -import { CollectionInfoWithSchema } from '@unique-nft/sdk/tokens'; +import { CollectionInfoWithSchema, TokenPropertiesResult, UniqueTokenDecoded } from '@unique-nft/sdk/tokens'; import '@unique-nft/sdk/tokens'; // need this to get sdk.collections import ImplementorAPI from './implementorAPI'; @@ -20,9 +19,7 @@ export class ImplementOpalAPI extends ImplementorAPI { } async impGetCollectionSdk(collectionId): Promise { - const result = await this.sdk.collections.get_new({ collectionId }); - - return result; + return this.sdk.collections.get_new({ collectionId }); } async impGetCollectionCount() { @@ -30,12 +27,15 @@ export class ImplementOpalAPI extends ImplementorAPI { return collectionStats?.created.toNumber(); } - async impGetToken(collectionId, tokenId): Promise { - const tokenData = await this.api.rpc.unique.tokenData(collectionId, tokenId); - return tokenData || null; + async impGetToken(collectionId, tokenId): Promise { + return this.sdk.tokens.get_new({ collectionId, tokenId }); } async impGetTokenCount(collectionId) { return (await this.api.rpc.unique.lastTokenId(collectionId)).toNumber(); } + + async impGetTokenPropertiesSdk(collectionId, tokenId): Promise { + return this.sdk.tokens.properties({ collectionId, tokenId }); + } } diff --git a/lib/token/tokenData.ts b/lib/token/tokenData.ts index 2456afd5..4a0c1cf2 100644 --- a/lib/token/tokenData.ts +++ b/lib/token/tokenData.ts @@ -1,103 +1,60 @@ +/* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable no-underscore-dangle */ import { ICollectionSchemaInfo } from 'crawlers/crawlers.interfaces'; -import { UpDataStructsTokenData } from '@unique-nft/unique-mainnet-types'; -import { normalizeSubstrateAddress } from '../../utils/utils'; -import protobuf from '../../utils/protobuf'; +import { TokenPropertiesResult, UniqueTokenDecoded } from '@unique-nft/sdk/tokens'; +import { normalizeSubstrateAddress, sanitizePropertiesValues } from '../../utils/utils'; import { ITokenDbEntity } from './tokenDbEntity.interface'; import { OpalAPI } from '../providerAPI/bridgeProviderAPI/concreate/opalAPI'; -function parseConstDataValue(constData, schema) { - const buffer = Buffer.from(constData.replace('0x', ''), 'hex'); - if (buffer.toString().length !== 0 && constData.replace('0x', '') && schema !== null) { - return { - constData, - buffer, - locale: 'en', - root: schema, - }; - } - - return { constData }; -} - -function getDeserializeConstData(statement) { - let result: { hex?: string } = {}; - if ('buffer' in statement) { - try { - result = { ...protobuf.deserializeNFT(statement) }; - } catch (error) { - // todo: Useless log now. Should log error by logger with collectionId and tokenId. - // eslint-disable-next-line no-console - console.error( - 'getDeserializeConstData(): Could not process constData with existing schema.', - // statement, - `Error message: '${error?.message}'`, - ); - result.hex = statement.constData?.toString().replace('0x', '') || statement.constData; - } - } else { - result.hex = statement.constData?.toString().replace('0x', '') || statement.constData; - } - - return result; -} - -function processConstData(constData, schema) { - if (!constData) { - return {}; - } - - const statement = parseConstDataValue(constData, schema); - return getDeserializeConstData(statement); -} - -function processProperties(schema: any, rawToken: UpDataStructsTokenData) - : { data: Object, properties: Object } { - const rawProperties = rawToken.properties; - - const properties : { - _old_constData?: Object | string, - } = {}; - - rawProperties.forEach(({ key, value }) => { - const strKey = key.toUtf8(); - const strValue = value.toUtf8(); - let processedValue = null; - - if (['_old_constData'].includes(strKey)) { - try { processedValue = value.toHex(); } catch (err) { /* */ } - } - - properties[strKey] = processedValue || strValue; - }); - - return { - data: processConstData(properties._old_constData, schema), - properties, - }; -} - -function formatTokenData(tokenId: number, collectionInfo: ICollectionSchemaInfo, rawToken: UpDataStructsTokenData) +function formatTokenData({ + tokenDecoded, + tokenProperties +}: { + tokenDecoded: UniqueTokenDecoded, + tokenProperties: TokenPropertiesResult +}) : ITokenDbEntity { - const { collectionId, schema } = collectionInfo; - - const rawOwnerJson = rawToken.owner.toJSON() as { substrate?: string, ethereum?: string }; - - const owner = rawOwnerJson?.substrate || rawOwnerJson?.ethereum; + const { + tokenId: token_id, + collectionId: collection_id, + image, + attributes, + nestingParentToken, + } = tokenDecoded; + + const { + owner: rawOwner, + }: { owner: { Ethereum?: string; Substrate?: string } } = tokenDecoded; + + const owner = rawOwner?.Ethereum || rawOwner?.Substrate; + + let parentId = null; + if (nestingParentToken) { + const { collectionId, tokenId } = nestingParentToken as { collectionId: number; tokenId: number }; + parentId = `${collectionId}_${tokenId}`; + } return { - token_id: tokenId, - collection_id: collectionId, + token_id, + collection_id, owner, owner_normalized: normalizeSubstrateAddress(owner), - ...processProperties(schema, rawToken), + data: { image }, + attributes: JSON.stringify(attributes), + properties: tokenProperties + ? JSON.stringify(sanitizePropertiesValues(tokenProperties)) + : '[]', + parent_id: parentId, }; } export async function getFormattedToken(tokenId: number, collectionInfo: ICollectionSchemaInfo, bridgeAPI: OpalAPI) : Promise { const { collectionId } = collectionInfo; - const rawToken = await bridgeAPI.getToken(collectionId, tokenId); + const { tokenDecoded, tokenProperties } = await bridgeAPI.getToken(collectionId, tokenId); - return rawToken ? formatTokenData(tokenId, collectionInfo, rawToken) : null; + return tokenDecoded ? formatTokenData({ + tokenDecoded, + tokenProperties + }) : null; } diff --git a/lib/token/tokenDb.ts b/lib/token/tokenDb.ts index 055ae0ea..15f280b4 100644 --- a/lib/token/tokenDb.ts +++ b/lib/token/tokenDb.ts @@ -17,6 +17,8 @@ const TOKEN_FIELDS = [ 'data', 'date_of_creation', 'parent_id', + 'properties', + 'attributes' ]; function prepareQueryReplacements(token: ITokenDbEntity) { diff --git a/lib/token/tokenDbEntity.interface.ts b/lib/token/tokenDbEntity.interface.ts index df212713..a30879de 100644 --- a/lib/token/tokenDbEntity.interface.ts +++ b/lib/token/tokenDbEntity.interface.ts @@ -6,4 +6,7 @@ export interface ITokenDbEntity { owner_normalized: string, data: Object, date_of_creation?: number, + properties: string, + attributes: string, + parent_id: string | null } diff --git a/utils/utils.js b/utils/utils.js index d2ee35d2..d4286715 100644 --- a/utils/utils.js +++ b/utils/utils.js @@ -187,7 +187,15 @@ function getTokenIdFromNestingAddress(address) { } function sanitizeUnicodeString(str) { - return str.replace(/\\u0000/g, ''); + // eslint-disable-next-line no-control-regex + return str.replace(/\\u0000|\x00/g, ''); +} + +function sanitizePropertiesValues(propertiesArr) { + return propertiesArr.map(({ key, value }) => ({ + key, + value: sanitizeUnicodeString(value), + })); } module.exports = { @@ -210,5 +218,6 @@ module.exports = { isNestingAddress, getCollectionIdFromNestingAddress, getTokenIdFromNestingAddress, - sanitizeUnicodeString + sanitizeUnicodeString, + sanitizePropertiesValues }; From 2b465192c4bd7a19ee9260d9dd8a7056100cd5e6 Mon Sep 17 00:00:00 2001 From: Sergey Chernetsky Date: Fri, 5 Aug 2022 14:15:59 +0800 Subject: [PATCH 3/4] Minor fix --- lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts b/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts index 9b4b35d1..69f7c2b9 100644 --- a/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts +++ b/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts @@ -4,7 +4,7 @@ import { UpDataStructsRpcCollection } from '@unique-nft/unique-mainnet-types'; import { CollectionInfoWithSchema, TokenPropertiesResult, UniqueTokenDecoded } from '@unique-nft/sdk/tokens'; -import '@unique-nft/sdk/tokens'; // need this to get sdk.collections +import '@unique-nft/sdk/tokens'; // need this to get sdk.collections and sdk.tokens declarations import ImplementorAPI from './implementorAPI'; export class ImplementOpalAPI extends ImplementorAPI { From 3945d0360244f7607093e5256ca4e0307e12b865 Mon Sep 17 00:00:00 2001 From: Sergey Chernetsky Date: Fri, 5 Aug 2022 18:18:52 +0800 Subject: [PATCH 4/4] Back to old data format --- .../bridgeProviderAPI/concreate/opalAPI.ts | 6 +- .../implement/implementOpalAPI.ts | 10 +- lib/token/tokenData.ts | 97 +++++++++++++++++-- 3 files changed, 101 insertions(+), 12 deletions(-) diff --git a/lib/providerAPI/bridgeProviderAPI/concreate/opalAPI.ts b/lib/providerAPI/bridgeProviderAPI/concreate/opalAPI.ts index 30a012c5..f630a0ab 100644 --- a/lib/providerAPI/bridgeProviderAPI/concreate/opalAPI.ts +++ b/lib/providerAPI/bridgeProviderAPI/concreate/opalAPI.ts @@ -2,6 +2,7 @@ import { CollectionInfoWithSchema, TokenPropertiesResult, UniqueTokenDecoded } f import { UpDataStructsCollectionLimits, UpDataStructsRpcCollection, + UpDataStructsTokenData, } from '@unique-nft/unique-mainnet-types'; import { ImplementOpalAPI } from '../implement/implementOpalAPI'; import AbstractAPI from './abstractAPI'; @@ -30,15 +31,18 @@ export class OpalAPI extends AbstractAPI { } async getToken(collectionId, tokenId): Promise<{ + rawToken: UpDataStructsTokenData | null, tokenDecoded: UniqueTokenDecoded | null, tokenProperties: TokenPropertiesResult | null }> { - const [tokenDecoded, tokenProperties] = await Promise.all([ + const [rawToken, tokenDecoded, tokenProperties] = await Promise.all([ this.impl.impGetToken(collectionId, tokenId), + this.impl.impGetTokenSdk(collectionId, tokenId), this.impl.impGetTokenPropertiesSdk(collectionId, tokenId) ]); return { + rawToken, tokenDecoded, tokenProperties }; diff --git a/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts b/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts index 69f7c2b9..417b61b7 100644 --- a/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts +++ b/lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts @@ -1,7 +1,8 @@ /* eslint-disable import/no-duplicates */ import { UpDataStructsCollectionLimits, - UpDataStructsRpcCollection + UpDataStructsRpcCollection, + UpDataStructsTokenData } from '@unique-nft/unique-mainnet-types'; import { CollectionInfoWithSchema, TokenPropertiesResult, UniqueTokenDecoded } from '@unique-nft/sdk/tokens'; import '@unique-nft/sdk/tokens'; // need this to get sdk.collections and sdk.tokens declarations @@ -27,7 +28,12 @@ export class ImplementOpalAPI extends ImplementorAPI { return collectionStats?.created.toNumber(); } - async impGetToken(collectionId, tokenId): Promise { + async impGetToken(collectionId, tokenId): Promise { + const tokenData = await this.api.rpc.unique.tokenData(collectionId, tokenId); + return tokenData || null; + } + + async impGetTokenSdk(collectionId, tokenId): Promise { return this.sdk.tokens.get_new({ collectionId, tokenId }); } diff --git a/lib/token/tokenData.ts b/lib/token/tokenData.ts index 4a0c1cf2..9e2dfdb1 100644 --- a/lib/token/tokenData.ts +++ b/lib/token/tokenData.ts @@ -1,23 +1,100 @@ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable no-underscore-dangle */ +import { UpDataStructsTokenData } from '@unique-nft/unique-mainnet-types'; import { ICollectionSchemaInfo } from 'crawlers/crawlers.interfaces'; import { TokenPropertiesResult, UniqueTokenDecoded } from '@unique-nft/sdk/tokens'; -import { normalizeSubstrateAddress, sanitizePropertiesValues } from '../../utils/utils'; +import { + normalizeSubstrateAddress, + sanitizePropertiesValues +} from '../../utils/utils'; +import protobuf from '../../utils/protobuf'; import { ITokenDbEntity } from './tokenDbEntity.interface'; import { OpalAPI } from '../providerAPI/bridgeProviderAPI/concreate/opalAPI'; +function parseConstDataValue(constData, schema) { + const buffer = Buffer.from(constData.replace('0x', ''), 'hex'); + if (buffer.toString().length !== 0 && constData.replace('0x', '') && schema !== null) { + return { + constData, + buffer, + locale: 'en', + root: schema, + }; + } + + return { constData }; +} + +function getDeserializeConstData(statement) { + let result: { hex?: string } = {}; + if ('buffer' in statement) { + try { + result = { ...protobuf.deserializeNFT(statement) }; + } catch (error) { + // todo: Useless log now. Should log error by logger with collectionId and tokenId. + // eslint-disable-next-line no-console + console.error( + 'getDeserializeConstData(): Could not process constData with existing schema.', + // statement, + `Error message: '${error?.message}'`, + ); + result.hex = statement.constData?.toString().replace('0x', '') || statement.constData; + } + } else { + result.hex = statement.constData?.toString().replace('0x', '') || statement.constData; + } + + return result; +} + +function processConstData(constData, schema) { + if (!constData) { + return {}; + } + + const statement = parseConstDataValue(constData, schema); + return getDeserializeConstData(statement); +} + +function processOldProperties(schema: any, rawToken: UpDataStructsTokenData) + : { data: Object } { + const rawProperties = rawToken.properties; + + let oldConstData: Object | string | null = null; + + rawProperties.forEach(({ key, value }) => { + const strKey = key.toUtf8(); + const strValue = value.toUtf8(); + let processedValue = null; + + if (['_old_constData'].includes(strKey)) { + try { processedValue = value.toHex(); } catch (err) { /* */ } + + oldConstData = processedValue || strValue; + } + }); + + return { + data: processConstData(oldConstData, schema), + }; +} + function formatTokenData({ + rawToken, tokenDecoded, - tokenProperties + tokenProperties, + collectionInfo }: { + rawToken: UpDataStructsTokenData, tokenDecoded: UniqueTokenDecoded, - tokenProperties: TokenPropertiesResult -}) - : ITokenDbEntity { + tokenProperties: TokenPropertiesResult, + collectionInfo: ICollectionSchemaInfo +}) : ITokenDbEntity { + const { schema } = collectionInfo; + const { tokenId: token_id, collectionId: collection_id, - image, attributes, nestingParentToken, } = tokenDecoded; @@ -39,22 +116,24 @@ function formatTokenData({ collection_id, owner, owner_normalized: normalizeSubstrateAddress(owner), - data: { image }, attributes: JSON.stringify(attributes), properties: tokenProperties ? JSON.stringify(sanitizePropertiesValues(tokenProperties)) : '[]', parent_id: parentId, + ...processOldProperties(schema, rawToken), }; } export async function getFormattedToken(tokenId: number, collectionInfo: ICollectionSchemaInfo, bridgeAPI: OpalAPI) : Promise { const { collectionId } = collectionInfo; - const { tokenDecoded, tokenProperties } = await bridgeAPI.getToken(collectionId, tokenId); + const { rawToken, tokenDecoded, tokenProperties } = await bridgeAPI.getToken(collectionId, tokenId); return tokenDecoded ? formatTokenData({ + rawToken, tokenDecoded, - tokenProperties + tokenProperties, + collectionInfo }) : null; }