From fe75a3b60423361e55a8730a0ecab7493909c7c4 Mon Sep 17 00:00:00 2001 From: Stepan Lavrentev Date: Tue, 21 May 2024 22:31:30 +0300 Subject: [PATCH] adding VARA dictionary --- vara/.gitignore | 54 +++++++++++++++++++ vara/.project-cid | 1 + vara/README.md | 28 ++++++++++ vara/docker-compose.yml | 45 ++++++++++++++++ vara/package.json | 32 ++++++++++++ vara/project.yaml | 30 +++++++++++ vara/schema.graphql | 21 ++++++++ vara/scripts/publish-manifest.sh | 6 +++ vara/src/index.ts | 3 ++ vara/src/mappings/mappingHandlers.ts | 77 ++++++++++++++++++++++++++++ vara/tsconfig.json | 18 +++++++ 11 files changed, 315 insertions(+) create mode 100644 vara/.gitignore create mode 100644 vara/.project-cid create mode 100644 vara/README.md create mode 100644 vara/docker-compose.yml create mode 100644 vara/package.json create mode 100644 vara/project.yaml create mode 100644 vara/schema.graphql create mode 100644 vara/scripts/publish-manifest.sh create mode 100644 vara/src/index.ts create mode 100644 vara/src/mappings/mappingHandlers.ts create mode 100644 vara/tsconfig.json diff --git a/vara/.gitignore b/vara/.gitignore new file mode 100644 index 0000000..7bc6395 --- /dev/null +++ b/vara/.gitignore @@ -0,0 +1,54 @@ +# These are some examples of commonly ignored file patterns. +# You should customize this list as applicable to your project. +# Learn more about .gitignore: +# https://www.atlassian.com/git/tutorials/saving-changes/gitignore + +# Node artifact files +node_modules/ +dist/ +yarn.lock + +# Compiled Java class files +*.class + +# Compiled Python bytecode +*.py[cod] + +# Log files +*.log + +# Package files +*.jar + +# Maven +target/ +dist/ +src/types + +# JetBrains IDE +.idea/ + +# Unit test reports +TEST*.xml + +# Generated by MacOS +.DS_Store + +# Generated by Windows +Thumbs.db + +# Applications +*.app +*.exe +*.war + +# Large media files +*.mp4 +*.tiff +*.avi +*.flv +*.mov +*.wmv + +.data + diff --git a/vara/.project-cid b/vara/.project-cid new file mode 100644 index 0000000..9e9bac6 --- /dev/null +++ b/vara/.project-cid @@ -0,0 +1 @@ +QmaaVanbrtAB8wCEYrcgZp72jPpWHt5gh9zRfKE7py7286 \ No newline at end of file diff --git a/vara/README.md b/vara/README.md new file mode 100644 index 0000000..1237e25 --- /dev/null +++ b/vara/README.md @@ -0,0 +1,28 @@ +# SubQuery - Dictionary + +This special SubQuery Project provides a dictionary of data that pre-indexes events on chain to dramatically improve indexing the performance of your own SubQuery Project, sometimes up to 10x faster. + +It scans over the network, and simply records the module and method for every event/extrinsic on each block - please see the standard entities in `schema.graphql`. + +**If you want to create your SubQuery Dictionary to speed up indexing of your own Substrate chain, fork this project and let us know** + +# Geting Started +### 1. Install dependencies +```shell +yarn +``` + +### 2. Generate types +```shell +yarn codegen +``` + +### 3. Build +```shell +yarn build +``` + +### 4. Run locally +```shell +yarn start:docker +``` diff --git a/vara/docker-compose.yml b/vara/docker-compose.yml new file mode 100644 index 0000000..ba1834d --- /dev/null +++ b/vara/docker-compose.yml @@ -0,0 +1,45 @@ +version: '3' + +services: + postgres: + image: postgres:12-alpine + ports: + - 5432:5432 + volumes: + - .data/postgres:/var/lib/postgresql/data + environment: + POSTGRES_PASSWORD: postgres + + subquery-node: + image: onfinality/subql-node:latest + depends_on: + - "postgres" + restart: always + environment: + DB_USER: postgres + DB_PASS: postgres + DB_DATABASE: postgres + DB_HOST: postgres + DB_PORT: 5432 + volumes: + - ./:/app + command: + - -f=/app + - --disable-historical=true + + graphql-engine: + image: onfinality/subql-query:latest + ports: + - 3000:3000 + depends_on: + - "postgres" + restart: always + environment: + DB_USER: postgres + DB_PASS: postgres + DB_DATABASE: postgres + DB_HOST: postgres + DB_PORT: 5432 + command: + - --name=app + - --playground diff --git a/vara/package.json b/vara/package.json new file mode 100644 index 0000000..f31822d --- /dev/null +++ b/vara/package.json @@ -0,0 +1,32 @@ +{ + "name": "polkadot-dictionary", + "version": "1.0.0", + "description": "A SubQuery Dictionary Project that provides increased indexing speed to all projects", + "main": "dist/index.js", + "scripts": { + "build": "subql build", + "prepack": "rm -rf dist && npm build", + "test": "jest", + "codegen": "./node_modules/.bin/subql codegen", + "publish:all": "bash scripts/publish-manifest.sh", + "start:docker": "docker-compose pull && docker-compose up --remove-orphans" + }, + "homepage": "https://github.com/subquery/subql-dictionary", + "repository": "github:subquery/subql-dictionary", + "files": [ + "dist", + "schema.graphql", + "project.yaml" + ], + "author": "SubQuery Network", + "license": "Apache-2.0", + "devDependencies": { + "@polkadot/api": "^10", + "@subql/types": "latest", + "typescript": "^4.1.3", + "@subql/cli": "latest" + }, + "resolutions": { + "ipfs-unixfs": "6.0.6" + } +} diff --git a/vara/project.yaml b/vara/project.yaml new file mode 100644 index 0000000..e006195 --- /dev/null +++ b/vara/project.yaml @@ -0,0 +1,30 @@ +specVersion: 1.0.0 +name: SubQuery Dictionary - Vara +version: 1.0.0 +runner: + node: + name: '@subql/node' + version: '>=3.0.0' + options: + unfinalizedBlocks: false + historical: false + query: + name: '@subql/query' + version: '*' +description: >- + A SubQuery Dictionary Project that provides increased indexing speed to all + projects +repository: 'https://github.com/subquery/subql-dictionary' +schema: + file: ./schema.graphql +network: + chainId: '0xfe1b4c55fd4d668101126434206571a7838a8b6b93a6d1b95d607e78e6c53763' + endpoint: 'wss://archive-rpc.vara-network.io' +dataSources: + - kind: substrate/Runtime + startBlock: 1 + mapping: + file: ./dist/index.js + handlers: + - handler: handleBlock + kind: substrate/BlockHandler diff --git a/vara/schema.graphql b/vara/schema.graphql new file mode 100644 index 0000000..3c0fd50 --- /dev/null +++ b/vara/schema.graphql @@ -0,0 +1,21 @@ +type SpecVersion @entity { + id: ID! #specVersion + blockHeight: BigInt! +} + +type Event @entity { + id: ID! + module: String! @index + event: String! @index + blockHeight: BigInt! @index +} + +type Extrinsic @entity { + id: ID! + txHash: String! + module: String! @index + call: String! @index + blockHeight: BigInt! @index + success: Boolean! + isSigned: Boolean! +} diff --git a/vara/scripts/publish-manifest.sh b/vara/scripts/publish-manifest.sh new file mode 100644 index 0000000..26cd1e3 --- /dev/null +++ b/vara/scripts/publish-manifest.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +for file in ./project*.yaml; do + echo "Publishing $file..." + npx subql publish -f "$file" +done \ No newline at end of file diff --git a/vara/src/index.ts b/vara/src/index.ts new file mode 100644 index 0000000..5091615 --- /dev/null +++ b/vara/src/index.ts @@ -0,0 +1,3 @@ +//Exports all handler functions +export * from "./mappings/mappingHandlers"; +import "@polkadot/api-augment"; diff --git a/vara/src/mappings/mappingHandlers.ts b/vara/src/mappings/mappingHandlers.ts new file mode 100644 index 0000000..29821d9 --- /dev/null +++ b/vara/src/mappings/mappingHandlers.ts @@ -0,0 +1,77 @@ +import { EventRecord } from "@polkadot/types/interfaces"; +import { SubstrateExtrinsic, SubstrateBlock } from "@subql/types"; +import { SpecVersion, Event, Extrinsic } from "../types"; + +export async function handleBlock(block: SubstrateBlock): Promise { + // Initialise Spec Version + let specVersion = await SpecVersion.get(block.specVersion.toString()); + + // Check for updates to Spec Version + if (!specVersion) { + specVersion = new SpecVersion(block.specVersion.toString(),block.block.header.number.toBigInt()); + await specVersion.save(); + } + + // Process all events in block + const events = block.events + .filter( + (evt) => + !(evt.event.section === "system" && + evt.event.method === "ExtrinsicSuccess") + ) + .map((evt, idx) => + handleEvent(block.block.header.number.toString(), idx, evt) + ); + + // Process all calls in block + const calls = wrapExtrinsics(block).map((ext, idx) => + handleCall(`${block.block.header.number.toString()}-${idx}`, ext) + ); + + // Save all data + // All save order should always follow this structure + for (const event of events) { + await event.save() + } + for (const call of calls) { + await call.save() + } +} + +function handleEvent( + blockNumber: string, + eventIdx: number, + event: EventRecord +): Event { + const newEvent = new Event(`${blockNumber}-${eventIdx}`,event.event.section,event.event.method,BigInt(blockNumber)); + return newEvent; +} + +function handleCall(idx: string, extrinsic: SubstrateExtrinsic): Extrinsic { + const newExtrinsic = new Extrinsic( + idx, + extrinsic.extrinsic.hash.toString(), + extrinsic.extrinsic.method.section, + extrinsic.extrinsic.method.method, + extrinsic.block.block.header.number.toBigInt(), + extrinsic.success, + extrinsic.extrinsic.isSigned + ); + return newExtrinsic; +} + +function wrapExtrinsics(wrappedBlock: SubstrateBlock): SubstrateExtrinsic[] { + return wrappedBlock.block.extrinsics.map((extrinsic, idx) => { + const events = wrappedBlock.events.filter( + ({ phase }) => phase.isApplyExtrinsic && phase.asApplyExtrinsic.eqn(idx) + ); + return { + idx, + extrinsic, + block: wrappedBlock, + events, + success: + events.findIndex((evt) => evt.event.method === "ExtrinsicSuccess") > -1, + }; + }); +} diff --git a/vara/tsconfig.json b/vara/tsconfig.json new file mode 100644 index 0000000..9b95212 --- /dev/null +++ b/vara/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "emitDecoratorMetadata": true, + "experimentalDecorators": true, + "esModuleInterop": true, + "declaration": true, + "importHelpers": true, + "resolveJsonModule": true, + "module": "commonjs", + "outDir": "dist", + "rootDir": "src", + "target": "es2017" + }, + "include": [ + "src/**/*", + "node_modules/@subql/types/dist/global.d.ts" + ] +}