From 3823fa73319f9a7a21ea7d066ec9b39f310e58cf Mon Sep 17 00:00:00 2001 From: Angel Beltran Date: Wed, 24 Dec 2025 00:55:22 -0700 Subject: [PATCH] raw_store_events store started --- database_migrations/000001_initial.down.sql | 1 + database_migrations/000001_initial.up.sql | 10 ++ .../000002_processed_tables.down.sql | 3 + .../000002_processed_tables.up.sql | 29 ++++++ diagrams/event_tables.svg | 2 +- diagrams/event_tables.uml | 72 ++++++-------- diagrams/event_tables_draft_1.svg | 1 + diagrams/event_tables_draft_1.uml | 56 +++++++++++ go.mod | 11 +++ go.sum | 28 ++++++ internal/domains/raw_events/.events.go.swp | Bin 0 -> 12288 bytes internal/domains/raw_events/events.go | 89 ++++++++++++++++++ internal/webhooks/etsy/webhooks.go | 31 +++++- internal/webhooks/tiktok/webhooks.go | 31 +++++- internal/webhooks/webhooks.go | 9 +- internal/webhooks/wix/webhooks.go | 31 +++++- main.go | 32 ++++--- 17 files changed, 363 insertions(+), 73 deletions(-) create mode 100644 database_migrations/000001_initial.down.sql create mode 100644 database_migrations/000001_initial.up.sql create mode 100644 database_migrations/000002_processed_tables.down.sql create mode 100644 database_migrations/000002_processed_tables.up.sql create mode 100644 diagrams/event_tables_draft_1.svg create mode 100644 diagrams/event_tables_draft_1.uml create mode 100644 go.sum create mode 100644 internal/domains/raw_events/.events.go.swp create mode 100644 internal/domains/raw_events/events.go diff --git a/database_migrations/000001_initial.down.sql b/database_migrations/000001_initial.down.sql new file mode 100644 index 0000000..df4537d --- /dev/null +++ b/database_migrations/000001_initial.down.sql @@ -0,0 +1 @@ +DROP TABLE raw_store_events; diff --git a/database_migrations/000001_initial.up.sql b/database_migrations/000001_initial.up.sql new file mode 100644 index 0000000..a4f2f39 --- /dev/null +++ b/database_migrations/000001_initial.up.sql @@ -0,0 +1,10 @@ +CREATE TABLE raw_store_events ( + platform TEXT NOT NULL, + store_id TEXT NOT NULL, + event_timestamp TIMESTAMPTZ NOT NULL, + event_id TEXT NOT NULL, + raw_payload JSONB NOT NULL, + parsed BOOLEAN NOT NULL DEFAULT FALSE, + + PRIMARY KEY (platform, store_id, event_id, event_timestamp) +); diff --git a/database_migrations/000002_processed_tables.down.sql b/database_migrations/000002_processed_tables.down.sql new file mode 100644 index 0000000..1204a6e --- /dev/null +++ b/database_migrations/000002_processed_tables.down.sql @@ -0,0 +1,3 @@ +DROP TABLE tiktok_store_events; +DROP TABLE etsy_store_events; +DROP TABLE wix_store_events; diff --git a/database_migrations/000002_processed_tables.up.sql b/database_migrations/000002_processed_tables.up.sql new file mode 100644 index 0000000..6581455 --- /dev/null +++ b/database_migrations/000002_processed_tables.up.sql @@ -0,0 +1,29 @@ +CREATE TABLE tiktok_store_events ( + platform TEXT NOT NULL DEFAULT 'tiktok' CHECK (platform = 'tiktok'), + store_id TEXT NOT NULL, + event_timestamp TIMESTAMPTZ NOT NULL, + event_id TEXT NOT NULL, + + PRIMARY KEY (store_id, event_id, event_timestamp), + FOREIGN KEY (platform, store_id, event_id, event_timestamp) REFERENCES raw_store_events +); + +CREATE TABLE etsy_store_events ( + platform TEXT NOT NULL DEFAULT 'etsy' CHECK (platform = 'etsy'), + store_id TEXT NOT NULL, + event_timestamp TIMESTAMPTZ NOT NULL, + event_id TEXT NOT NULL, + + PRIMARY KEY (store_id, event_id, event_timestamp), + FOREIGN KEY (platform, store_id, event_id, event_timestamp) REFERENCES raw_store_events +); + +CREATE TABLE wix_store_events ( + platform TEXT NOT NULL DEFAULT 'wix' CHECK (platform = 'wix'), + store_id TEXT NOT NULL, + event_timestamp TIMESTAMPTZ NOT NULL, + event_id TEXT NOT NULL, + + PRIMARY KEY (store_id, event_id, event_timestamp), + FOREIGN KEY (platform, store_id, event_id, event_timestamp) REFERENCES raw_store_events +); diff --git a/diagrams/event_tables.svg b/diagrams/event_tables.svg index 6db4ff4..7a11097 100644 --- a/diagrams/event_tables.svg +++ b/diagrams/event_tables.svg @@ -1 +1 @@ -store_eventsplatformstore_idevent_idevent_timestamptiktok_store_idtiktok_event_id<platform B>_store_id<platform B>_event_id<platform C>_store_id<platform C>_event_id<platform ..>_store_id<platform ..>_event_idraw payload?reference tiktok_store_event_detailsreference <platform B>_store_event_detailsreference <platform C>_store_event_detailsonly one reference can exist at a time(eg event CANNOT be both from tiktok AND etsy)tiktok_store_event_detailsstore_idevent_idevent_timestamp[attribute: value ...]<platform B>_store_event_detailsstore_idevent_idevent_timestamp[attribute: value ...]<platform C>_store_event_detailsstore_idevent_idevent_timestamp[attribute: value ...] \ No newline at end of file +raw_store_eventsplatform textstore_id textevent_id textevent_timestamp timestamptzraw_payload jsonbparsed booleanindicates if the raw_payload has been parsedand stored in one of the referencing tablestiktok_store_eventsstore_id textevent_id textevent_timestamp timestamptz[attribute: value ...]etsy_store_eventsstore_id textevent_id textevent_timestamp timestamptz[attribute: value ...]etc_store_eventsstore_id textevent_id textevent_timestamp timestamptz[attribute: value ...](store_id, event_id)(store_id, event_id)(store_id, event_id) \ No newline at end of file diff --git a/diagrams/event_tables.uml b/diagrams/event_tables.uml index 5ccaf47..fb1e9dd 100644 --- a/diagrams/event_tables.uml +++ b/diagrams/event_tables.uml @@ -1,56 +1,40 @@ @startuml -object store_events -store_events : platform -store_events : store_id -store_events : event_id -store_events : event_timestamp -store_events : tiktok_store_id -store_events : tiktok_event_id -store_events : _store_id -store_events : _event_id -store_events : _store_id -store_events : _event_id -store_events : _store_id -store_events : _event_id -store_events : raw payload? +object raw_store_events +raw_store_events : platform text +raw_store_events : store_id text +raw_store_events : event_id text +raw_store_events : event_timestamp timestamptz +raw_store_events : raw_payload jsonb +raw_store_events : parsed boolean -note right of store_events::"tiktok_event_id" - reference tiktok_store_event_details -end note -note right of store_events::"_event_id" - reference _store_event_details -end note -note right of store_events::"_event_id" - reference _store_event_details -end note -note right of store_events::"_event_id" - only one reference can exist at a time - (eg event CANNOT be both from tiktok AND etsy) +note right of raw_store_events::"parsed" + indicates if the raw_payload has been parsed + and stored in one of the referencing tables end note -object tiktok_store_event_details -tiktok_store_event_details : store_id -tiktok_store_event_details : event_id -tiktok_store_event_details : event_timestamp -tiktok_store_event_details : [attribute: value ...] +object tiktok_store_events +tiktok_store_events : store_id text +tiktok_store_events : event_id text +tiktok_store_events : event_timestamp timestamptz +tiktok_store_events : [attribute: value ...] -object "_store_event_details" as platform_b_specific_store_event_details -platform_b_specific_store_event_details : store_id -platform_b_specific_store_event_details : event_id -platform_b_specific_store_event_details : event_timestamp -platform_b_specific_store_event_details : [attribute: value ...] +object etsy_store_events +etsy_store_events : store_id text +etsy_store_events : event_id text +etsy_store_events : event_timestamp timestamptz +etsy_store_events : [attribute: value ...] -object "_store_event_details" as platform_c_specific_store_event_details -platform_c_specific_store_event_details : store_id -platform_c_specific_store_event_details : event_id -platform_c_specific_store_event_details : event_timestamp -platform_c_specific_store_event_details : [attribute: value ...] +object etc_store_events +etc_store_events : store_id text +etc_store_events : event_id text +etc_store_events : event_timestamp timestamptz +etc_store_events : [attribute: value ...] -store_events <|-- tiktok_store_event_details -store_events <|-- platform_b_specific_store_event_details -store_events <|-- platform_c_specific_store_event_details +raw_store_events <|-- tiktok_store_events : (store_id, event_id) +raw_store_events <|-- etsy_store_events : (store_id, event_id) +raw_store_events <|-- etc_store_events : (store_id, event_id) @enduml diff --git a/diagrams/event_tables_draft_1.svg b/diagrams/event_tables_draft_1.svg new file mode 100644 index 0000000..6db4ff4 --- /dev/null +++ b/diagrams/event_tables_draft_1.svg @@ -0,0 +1 @@ +store_eventsplatformstore_idevent_idevent_timestamptiktok_store_idtiktok_event_id<platform B>_store_id<platform B>_event_id<platform C>_store_id<platform C>_event_id<platform ..>_store_id<platform ..>_event_idraw payload?reference tiktok_store_event_detailsreference <platform B>_store_event_detailsreference <platform C>_store_event_detailsonly one reference can exist at a time(eg event CANNOT be both from tiktok AND etsy)tiktok_store_event_detailsstore_idevent_idevent_timestamp[attribute: value ...]<platform B>_store_event_detailsstore_idevent_idevent_timestamp[attribute: value ...]<platform C>_store_event_detailsstore_idevent_idevent_timestamp[attribute: value ...] \ No newline at end of file diff --git a/diagrams/event_tables_draft_1.uml b/diagrams/event_tables_draft_1.uml new file mode 100644 index 0000000..5ccaf47 --- /dev/null +++ b/diagrams/event_tables_draft_1.uml @@ -0,0 +1,56 @@ +@startuml + + +object store_events +store_events : platform +store_events : store_id +store_events : event_id +store_events : event_timestamp +store_events : tiktok_store_id +store_events : tiktok_event_id +store_events : _store_id +store_events : _event_id +store_events : _store_id +store_events : _event_id +store_events : _store_id +store_events : _event_id +store_events : raw payload? + +note right of store_events::"tiktok_event_id" + reference tiktok_store_event_details +end note +note right of store_events::"_event_id" + reference _store_event_details +end note +note right of store_events::"_event_id" + reference _store_event_details +end note +note right of store_events::"_event_id" + only one reference can exist at a time + (eg event CANNOT be both from tiktok AND etsy) +end note + +object tiktok_store_event_details +tiktok_store_event_details : store_id +tiktok_store_event_details : event_id +tiktok_store_event_details : event_timestamp +tiktok_store_event_details : [attribute: value ...] + +object "_store_event_details" as platform_b_specific_store_event_details +platform_b_specific_store_event_details : store_id +platform_b_specific_store_event_details : event_id +platform_b_specific_store_event_details : event_timestamp +platform_b_specific_store_event_details : [attribute: value ...] + +object "_store_event_details" as platform_c_specific_store_event_details +platform_c_specific_store_event_details : store_id +platform_c_specific_store_event_details : event_id +platform_c_specific_store_event_details : event_timestamp +platform_c_specific_store_event_details : [attribute: value ...] + +store_events <|-- tiktok_store_event_details +store_events <|-- platform_b_specific_store_event_details +store_events <|-- platform_c_specific_store_event_details + + +@enduml diff --git a/go.mod b/go.mod index bd99d29..f436bec 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,14 @@ module ruben/inventory2 go 1.24.2 + +require github.com/jackc/pgx/v5 v5.7.6 + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/text v0.28.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8255168 --- /dev/null +++ b/go.sum @@ -0,0 +1,28 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/domains/raw_events/.events.go.swp b/internal/domains/raw_events/.events.go.swp new file mode 100644 index 0000000000000000000000000000000000000000..d01f698ea05877546d8b9e113216bfa165db55ca GIT binary patch literal 12288 zcmeI2O>7)B6vv&SiVhG!AS!V|&LGNe*v_0jgtmFq9ib0Cu&}SWZLkVh1*`&A0jq#jz$#!BunJfO{(B0jbcno)VI3-l_EJ#} z?JA2;TUZ6G0#*U5fK|XMU=^?mSOu&CRspMkRlq9nFI0eqgp`gD@}Yy|@&Et+-~Yef zN61ZZ1AGs@248~D!FBKfxB`~I0+uBs$H6gh6#RKFAwPm|!3KB-yanC_Z-6E6 z40r-O432__z+v#mFd?_VZ(tK_fGIErZr(%4FW_hJ4Y&?2fo1R_SOPUL1)c+k!RFnB z`~+@*ufPWQ2)qqm0}^xq1G8WX90!kpVQ>W80e-)Wkng}3;9al+&Vf3pfIGn<@E3I5 z1fPKo@G1BhXuaPD*T8$gK2`y%fK|XMU=^?m{O=W5Unh>_sw)$kM1$`!Ar8L#jjfLI zYElZ>D!DBd#XO`+(3H!TkR5ut%jH^?KGt(9luKDAPSnybbYvWEV|tZ1sB_jU_~p3z zx@F4NV!=YUN;4&+q+R0P`Mj!}8<~rkjYwPI;a*MO&#a#{jkzCZk;PG1r6Wb-eAMBY zVx82hXD@RpO1URT*TczftHCg;-xbl)wpvTMx8+S;LH5Wx!{t$)C&axu{ZVA&rZH1m z26Vl$Z575|UmeWP5q0 zLc|rlvam?eGx275%vr`u81=xSg+Ybxy>bJ|nc|eukSW$gSDit^0~Lwn;DxO1wg#sY z@g{@I-AI~oly~l$+b6IcK8`TjgG1-T*82*&VsV-1GrXr2tWY-erNISx?AtA{GIQmtj{ubNZCN30+)@Zj3#fWpPJ@d_0UY){MDZ&Sxv(z4I94UXxU& zrO|#C+Qz6NWVx8D^?ZgLCv0L@>GrLo=9rm&OzA&n3xnI^@G!m2fr5b#WK`q`35>)2 zk~EEj*%iyAIC1m=qo0eJz)E7hX`W@Q&9Rov3fwvgsNRd_VCXm1&*nX8af+ed#M-02 ze=Zaq@>7rG5p?fvptNpA5WPVLg(bPyZ8LpBVZ#xO!)-@ux!d$`E8f4zf=hvqp8m=i zU1-_d15LD}11wx5