From 4b178256a54d16ab16cc9339d16ca11faeab4739 Mon Sep 17 00:00:00 2001 From: Andrey Date: Thu, 12 Oct 2023 17:04:49 +0300 Subject: [PATCH 1/3] Add initial changes. --- moonstreamdb/moonstreamdb/models.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/moonstreamdb/moonstreamdb/models.py b/moonstreamdb/moonstreamdb/models.py index 3d7c25372..d175624bc 100644 --- a/moonstreamdb/moonstreamdb/models.py +++ b/moonstreamdb/moonstreamdb/models.py @@ -149,6 +149,7 @@ class EthereumLabel(Base): # type: ignore index=True, ) label_data = Column(JSONB, nullable=True) + block_hash = Column(VARCHAR(256), index=True) block_timestamp = Column(BigInteger, index=True) log_index = Column(Integer, nullable=True) created_at = Column( @@ -270,6 +271,7 @@ class PolygonLabel(Base): # type: ignore index=True, ) label_data = Column(JSONB, nullable=True) + block_hash = Column(VARCHAR(256), index=True) block_timestamp = Column(BigInteger, index=True) log_index = Column(Integer, nullable=True) created_at = Column( @@ -376,6 +378,7 @@ class MumbaiLabel(Base): # type: ignore index=True, ) label_data = Column(JSONB, nullable=True) + block_hash = Column(VARCHAR(256), index=True) block_timestamp = Column(BigInteger, index=True) log_index = Column(Integer, nullable=True) created_at = Column( @@ -486,6 +489,7 @@ class XDaiLabel(Base): # type: ignore index=True, ) label_data = Column(JSONB, nullable=True) + block_hash = Column(VARCHAR(256), index=True) block_timestamp = Column(BigInteger, index=True) log_index = Column(Integer, nullable=True) created_at = Column( @@ -607,6 +611,7 @@ class WyrmLabel(Base): # type: ignore index=True, ) label_data = Column(JSONB, nullable=True) + block_hash = Column(VARCHAR(256), index=True) block_timestamp = Column(BigInteger, index=True) log_index = Column(Integer, nullable=True) created_at = Column( @@ -738,6 +743,7 @@ class ZkSyncEraLabel(Base): # type: ignore index=True, ) label_data = Column(JSONB, nullable=True) + block_hash = Column(VARCHAR(256), index=True) block_timestamp = Column(BigInteger, index=True) log_index = Column(Integer, nullable=True) created_at = Column( @@ -869,6 +875,7 @@ class ZkSyncEraTestnetLabel(Base): # type: ignore index=True, ) label_data = Column(JSONB, nullable=True) + block_hash = Column(VARCHAR(256), index=True) block_timestamp = Column(BigInteger, index=True) log_index = Column(Integer, nullable=True) created_at = Column( From 93db4770f345f774408b4752d20a251ecb4a30b0 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 16 Oct 2023 14:50:01 +0300 Subject: [PATCH 2/3] Add migration. --- .../versions/91451b16c5c8_add_block_hash.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 moonstreamdb/alembic/versions/91451b16c5c8_add_block_hash.py diff --git a/moonstreamdb/alembic/versions/91451b16c5c8_add_block_hash.py b/moonstreamdb/alembic/versions/91451b16c5c8_add_block_hash.py new file mode 100644 index 000000000..db240ec7c --- /dev/null +++ b/moonstreamdb/alembic/versions/91451b16c5c8_add_block_hash.py @@ -0,0 +1,54 @@ +"""Add block_hash + +Revision ID: 91451b16c5c8 +Revises: 0f8ee1ebb45f +Create Date: 2023-10-16 14:39:31.687486 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '91451b16c5c8' +down_revision = '0f8ee1ebb45f' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('ethereum_labels', sa.Column('block_hash', sa.VARCHAR(length=256), nullable=True)) + op.create_index(op.f('ix_ethereum_labels_block_hash'), 'ethereum_labels', ['block_hash'], unique=False) + op.add_column('mumbai_labels', sa.Column('block_hash', sa.VARCHAR(length=256), nullable=True)) + op.create_index(op.f('ix_mumbai_labels_block_hash'), 'mumbai_labels', ['block_hash'], unique=False) + op.add_column('polygon_labels', sa.Column('block_hash', sa.VARCHAR(length=256), nullable=True)) + op.create_index(op.f('ix_polygon_labels_block_hash'), 'polygon_labels', ['block_hash'], unique=False) + op.add_column('wyrm_labels', sa.Column('block_hash', sa.VARCHAR(length=256), nullable=True)) + op.create_index(op.f('ix_wyrm_labels_block_hash'), 'wyrm_labels', ['block_hash'], unique=False) + op.add_column('xdai_labels', sa.Column('block_hash', sa.VARCHAR(length=256), nullable=True)) + op.create_index(op.f('ix_xdai_labels_block_hash'), 'xdai_labels', ['block_hash'], unique=False) + op.add_column('zksync_era_labels', sa.Column('block_hash', sa.VARCHAR(length=256), nullable=True)) + op.create_index(op.f('ix_zksync_era_labels_block_hash'), 'zksync_era_labels', ['block_hash'], unique=False) + op.add_column('zksync_era_testnet_labels', sa.Column('block_hash', sa.VARCHAR(length=256), nullable=True)) + op.create_index(op.f('ix_zksync_era_testnet_labels_block_hash'), 'zksync_era_testnet_labels', ['block_hash'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_zksync_era_testnet_labels_block_hash'), table_name='zksync_era_testnet_labels') + op.drop_column('zksync_era_testnet_labels', 'block_hash') + op.drop_index(op.f('ix_zksync_era_labels_block_hash'), table_name='zksync_era_labels') + op.drop_column('zksync_era_labels', 'block_hash') + op.drop_index(op.f('ix_xdai_labels_block_hash'), table_name='xdai_labels') + op.drop_column('xdai_labels', 'block_hash') + op.drop_index(op.f('ix_wyrm_labels_block_hash'), table_name='wyrm_labels') + op.drop_column('wyrm_labels', 'block_hash') + op.drop_index(op.f('ix_polygon_labels_block_hash'), table_name='polygon_labels') + op.drop_column('polygon_labels', 'block_hash') + op.drop_index(op.f('ix_mumbai_labels_block_hash'), table_name='mumbai_labels') + op.drop_column('mumbai_labels', 'block_hash') + op.drop_index(op.f('ix_ethereum_labels_block_hash'), table_name='ethereum_labels') + op.drop_column('ethereum_labels', 'block_hash') + # ### end Alembic commands ### From 0c08796d1e27a0c4838b0ebcc466ca8d61444000 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 18 Oct 2023 07:03:17 +0300 Subject: [PATCH 3/3] Add changes. --- crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py index a2c5f7412..11ca3a632 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py @@ -19,6 +19,7 @@ class Event: event_name: str args: Dict[str, Any] address: str + block_hash: str block_number: int block_timestamp: int transaction_hash: str @@ -131,6 +132,7 @@ def _crawl_events( event_name=raw_event["event"], args=raw_event["args"], address=raw_event["address"], + block_hash=raw_event["blockHash"], block_number=raw_event["blockNumber"], block_timestamp=raw_event["blockTimestamp"], transaction_hash=raw_event["transactionHash"], @@ -178,6 +180,7 @@ def _autoscale_crawl_events( event_name=raw_event["event"], args=raw_event["args"], address=raw_event["address"], + block_hash=raw_event["blockHash"], block_number=raw_event["blockNumber"], block_timestamp=raw_event["blockTimestamp"], transaction_hash=raw_event["transactionHash"],