From b19cb846405230fff84ca3dd505086d6e7eb902b Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Sun, 12 Jul 2026 21:45:07 +0530 Subject: [PATCH 1/9] test: add db schema types and test-kit factory primitives Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- .../src/api/public/v1/members/createMember.ts | 4 +- .../repositories/organizationRepository.ts | 28 +-- ...atabase-schema-types-as-source-of-truth.md | 61 ++++++ ...07-test-factory-primitives-and-defaults.md | 83 ++++++++ docs/adr/README.md | 2 + pnpm-lock.yaml | 46 +++-- .../src/activities/enrichment.ts | 26 ++- services/libs/data-access-layer/package.json | 4 +- .../data-access-layer/src/activities/sql.ts | 2 +- services/libs/data-access-layer/src/index.ts | 5 +- .../member-organization-affiliation/index.ts | 36 +++- .../src/member_segment_affiliations/index.ts | 89 +++++++-- .../data-access-layer/src/members/base.ts | 21 +- .../src/members/identities.ts | 59 +++--- .../src/members/organizations.ts | 124 ++++++++---- .../apps/data_sink_worker/repo/member.repo.ts | 11 +- .../data_sink_worker/repo/settings.repo.ts | 2 +- .../src/organizations/base.ts | 126 ++++++++++-- .../src/organizations/identities.ts | 124 +++++++----- .../data-access-layer/src/segments/index.ts | 88 ++++++++- services/libs/test-kit/package.json | 9 +- services/libs/test-kit/src/db/index.ts | 2 + .../libs/test-kit/src/{ => db}/postgres.ts | 7 +- .../test-kit/src/{fixtures.ts => db/qx.ts} | 0 .../src/factories/activity-relation.ts | 172 ++++++++++++++++ .../libs/test-kit/src/factories/defaults.ts | 30 +++ services/libs/test-kit/src/factories/index.ts | 10 + .../test-kit/src/factories/member-identity.ts | 25 +++ ...ember-organization-affiliation-override.ts | 26 +++ .../src/factories/member-organization.ts | 25 +++ .../factories/member-segment-affiliation.ts | 25 +++ .../libs/test-kit/src/factories/member.ts | 27 +++ .../src/factories/organization-identity.ts | 24 +++ .../test-kit/src/factories/organization.ts | 30 +++ .../libs/test-kit/src/factories/segment.ts | 185 ++++++++++++++++++ services/libs/test-kit/src/faker.ts | 1 + services/libs/test-kit/src/types.ts | 6 - .../libs/types/src/db/activity-relations.ts | 55 ++++++ services/libs/types/src/db/index.ts | 9 + .../libs/types/src/db/member-identities.ts | 36 ++++ ...mber-organization-affiliation-overrides.ts | 13 ++ .../libs/types/src/db/member-organizations.ts | 25 +++ .../src/db/member-segment-affiliations.ts | 17 ++ services/libs/types/src/db/members.ts | 29 +++ .../types/src/db/organization-identities.ts | 21 ++ services/libs/types/src/db/organizations.ts | 32 +++ services/libs/types/src/db/segments.ts | 35 ++++ services/libs/types/src/index.ts | 2 + services/libs/types/src/organizations.ts | 11 -- 49 files changed, 1598 insertions(+), 232 deletions(-) create mode 100644 docs/adr/0006-database-schema-types-as-source-of-truth.md create mode 100644 docs/adr/0007-test-factory-primitives-and-defaults.md create mode 100644 services/libs/test-kit/src/db/index.ts rename services/libs/test-kit/src/{ => db}/postgres.ts (98%) rename services/libs/test-kit/src/{fixtures.ts => db/qx.ts} (100%) create mode 100644 services/libs/test-kit/src/factories/activity-relation.ts create mode 100644 services/libs/test-kit/src/factories/defaults.ts create mode 100644 services/libs/test-kit/src/factories/index.ts create mode 100644 services/libs/test-kit/src/factories/member-identity.ts create mode 100644 services/libs/test-kit/src/factories/member-organization-affiliation-override.ts create mode 100644 services/libs/test-kit/src/factories/member-organization.ts create mode 100644 services/libs/test-kit/src/factories/member-segment-affiliation.ts create mode 100644 services/libs/test-kit/src/factories/organization-identity.ts create mode 100644 services/libs/test-kit/src/factories/organization.ts create mode 100644 services/libs/test-kit/src/factories/segment.ts create mode 100644 services/libs/test-kit/src/faker.ts delete mode 100644 services/libs/test-kit/src/types.ts create mode 100644 services/libs/types/src/db/activity-relations.ts create mode 100644 services/libs/types/src/db/index.ts create mode 100644 services/libs/types/src/db/member-identities.ts create mode 100644 services/libs/types/src/db/member-organization-affiliation-overrides.ts create mode 100644 services/libs/types/src/db/member-organizations.ts create mode 100644 services/libs/types/src/db/member-segment-affiliations.ts create mode 100644 services/libs/types/src/db/members.ts create mode 100644 services/libs/types/src/db/organization-identities.ts create mode 100644 services/libs/types/src/db/organizations.ts create mode 100644 services/libs/types/src/db/segments.ts diff --git a/backend/src/api/public/v1/members/createMember.ts b/backend/src/api/public/v1/members/createMember.ts index 075be732ee..e030774bf0 100644 --- a/backend/src/api/public/v1/members/createMember.ts +++ b/backend/src/api/public/v1/members/createMember.ts @@ -4,8 +4,8 @@ import { z } from 'zod' import { captureApiChange, memberCreateAction, memberEditIdentitiesAction } from '@crowd/audit-logs' import { getProperDisplayName } from '@crowd/common' import { - insertManyMemberIdentities, createMember as insertMember, + insertMemberIdentities, optionsQx, } from '@crowd/data-access-layer' import { MemberIdentityType } from '@crowd/types' @@ -52,7 +52,7 @@ export async function createMember(req: Request, res: Response): Promise { manuallyCreated: true, }) - const dbIdentities = await insertManyMemberIdentities( + const dbIdentities = await insertMemberIdentities( tx, identities.map((identity) => ({ ...identity, diff --git a/backend/src/database/repositories/organizationRepository.ts b/backend/src/database/repositories/organizationRepository.ts index 9f6a50b28c..fa19d24c80 100644 --- a/backend/src/database/repositories/organizationRepository.ts +++ b/backend/src/database/repositories/organizationRepository.ts @@ -16,7 +16,6 @@ import { IDbOrganization, OrgIdentityField, OrganizationField, - addOrgIdentity, addOrgsToSegments, cleanUpOrgIdentities, cleanupForOganization, @@ -27,6 +26,7 @@ import { findManyOrgAttributes, findOrgAttributes, findOrgById, + insertOrganizationIdentities, markOrgAttributeDefault, queryOrgIdentities, updateOrgIdentityVerifiedFlag, @@ -619,16 +619,22 @@ class OrganizationRepository { ): Promise { const qx = SequelizeRepository.getQueryExecutor(options) - await addOrgIdentity(qx, { - organizationId, - platform: identity.platform, - source: identity.source, - sourceId: identity.sourceId || null, - value: identity.value, - type: identity.type, - verified: identity.verified, - integrationId: identity.integrationId || null, - }) + await insertOrganizationIdentities( + qx, + [ + { + organizationId, + platform: identity.platform, + source: identity.source, + sourceId: identity.sourceId || null, + value: identity.value, + type: identity.type, + verified: identity.verified, + integrationId: identity.integrationId || null, + }, + ], + false, + ) } static async getIdentities( diff --git a/docs/adr/0006-database-schema-types-as-source-of-truth.md b/docs/adr/0006-database-schema-types-as-source-of-truth.md new file mode 100644 index 0000000000..1946afbc53 --- /dev/null +++ b/docs/adr/0006-database-schema-types-as-source-of-truth.md @@ -0,0 +1,61 @@ +# ADR-0006: Database schema types as the source of truth + +**Date**: 2026-07-09 +**Status**: accepted +**Deciders**: Yeganathan S + +## Context + +Working with entity types in CDP is messy. The same concepts appear as overlapping shapes across `@crowd/types`, DAL inputs, merge helpers, and API/domain models — often incomplete relative to the database, with inconsistent nullability and no single source of truth. That makes it hard to change data access safely, reuse types across packages, and keep tests aligned with production. + +We need a durable way to model persisted data that the rest of the codebase can build on, without inventing yet another parallel type family per feature. + +## Decision + +Treat the database schema as the source of truth for persisted entity shapes. Introduce a dedicated `db/` area under `@crowd/types` for schema-aligned row types, and derive narrower types (create/update payloads, factory options, domain views) from those via composition (`Pick`, `Partial`, `Omit`, intersections) rather than hand-maintaining duplicates. + +These types live in `@crowd/types`, not in the data-access layer: they are shared vocabulary for backend, workers, libraries, and test utilities. DAL remains responsible for queries and persistence; it consumes and returns these types instead of owning the canonical shapes. + +This applies to entities generally (members, identities, organizations, and others as we touch them), not only the first tables we migrate. + +## Conventions + +These rules exist so people (and tools) do not confuse the row type with write payloads, or nullability with “field may be omitted.” + +1. **One row type per table** — describes the entity as stored. Field nullability matches the schema. Database defaults do not make read fields optional. +2. **Write payloads are derived from the row type** — use `Pick`, `Partial`, `Omit`, and intersections instead of duplicating the full interface. +3. **Nullability ≠ optionality** — `| null` means the column can store null; `?` / `Partial` means the caller may omit the key. Both can appear; they mean different things. +4. **System-owned fields stay off write payloads** — things like tenant id, audit timestamps, and soft-delete markers are set by persistence code, not by every caller. +5. **Layers that supply defaults may widen further** — e.g. a test factory can take a partial write payload and fill required fields before calling DAL. That does not change the row type or the DAL contract. + +## Alternatives Considered + +### Alternative 1: Keep defining types ad hoc next to each feature +- **Pros**: Fast locally; no cross-package coordination. +- **Cons**: Duplicates and drift continue; nullability and field sets disagree across call sites. +- **Why not**: That is the current pain. It does not scale as more packages share the same entities. + +### Alternative 2: Own canonical entity types inside `@crowd/data-access-layer` +- **Pros**: Types sit next to SQL; easy for DAL authors to update. +- **Cons**: Anything that needs a row shape (test-kit, common services, workers, backend) must depend on DAL or re-copy types. +- **Why not**: Row shapes are shared contracts, not DAL implementation details. Putting them in `@crowd/types` keeps the dependency graph thin and matches how other shared models already live. + +### Alternative 3: Generate types from the schema (e.g. introspection tooling) +- **Pros**: Always in sync with migrations; less manual work. +- **Cons**: Tooling and review process are not in place; generated output can be noisy and hard to evolve with domain naming. +- **Why not**: Manual schema-aligned types are enough to establish the pattern now. Generation can be revisited once the convention is stable. + +## Consequences + +### Positive +- One place to look for “what does this table look like in TypeScript.” +- Downstream APIs and tests extend from the same base instead of inventing parallel models. +- Clearer package boundaries: `@crowd/types` for shapes, DAL for access, factories/services for composition. + +### Negative +- Existing aliases and legacy shapes (`MemberRow`, feature-local inputs, etc.) will coexist during migration. +- Authors must update `db/` types when migrations change columns or nullability. + +### Risks +- **Partial adoption** — mitigated by migrating types when a table is touched (factories, DAL hardening, new features), not a big-bang rewrite. +- **Drift from schema** — mitigated by checking live schema (or migrations) when adding or changing `db/` types; optional codegen later. diff --git a/docs/adr/0007-test-factory-primitives-and-defaults.md b/docs/adr/0007-test-factory-primitives-and-defaults.md new file mode 100644 index 0000000000..eaa5dc1f59 --- /dev/null +++ b/docs/adr/0007-test-factory-primitives-and-defaults.md @@ -0,0 +1,83 @@ +# ADR-0007: Test factory primitives and defaults + +**Date**: 2026-07-10 +**Status**: accepted +**Deciders**: Yeganathan S + +## Context + +Shared test factories are easy to turn into hidden scenario builders: they invent data the test never declared, blur what is under assertion, and drift from the real insert shapes used in production code (see ADR-0006). + +We need a simple, durable rule for how test data is created — what the factory does, what defaults may fill, and what the test must own — so the pattern does not divert as more factories are added. + +## Decision + +Treat test setup as **composition of sharp primitives**, not smart world-builders. Keep three layers separate. + +### 1. Primitives + +- One job: **persist what they are given**, preferably in bulk, via the DAL. +- Input stays close to schema insert types (`*DbInsert` and small, explicit extensions when a factory accepts related children in one call). +- No faker inside primitives. No silent graph scaffolding. No behavioral guesses. +- When a primitive accepts structured input (for example a hierarchy), it may **derive structural fields** from that shape (links, type/level, denormalized parent labels). That derivation is part of the primitive’s contract — not a default. + +### 2. Defaults (opt-in) + +- Separate helpers (`withXDefaults`) fill an **allowlist** of missing label / harmless system fields **on rows the caller already wrote**. +- Call sites opt in: `createX(qx, withXDefaults([...]))`. Defaults are never baked into primitives. +- Semantics: + - **Allowlist only** — unlisted fields are untouched. + - **Explicit always wins** — a provided value is never overwritten. + - **Only** `undefined` **is missing** — omitted / `undefined` may be filled; `null` **is intentional and sticks**. + - **Labels / harmless flags only** — e.g. generated `id`, display names, join timestamps, inactive-path status flags. Ask per field: *if we invent this, could we change which production branch the test hits?* If yes, it does not belong in defaults. + - **Caller owns scenario identity** when the value defines the case (names, slugs, platforms, dates, relationship targets, and any field that makes two cases different). + +Defaults do **not** build a realistic entity or graph. They only patch allowlisted gaps (e.g. `displayName`, `joinedAt`). Identities, org stints, activities, and other scenario data stay with the test. + +### 3. Test composition + +- The suite owns behavioral fields and graph shape. +- Prefer inline fixtures and spreads in the test file. +- File-local seed helpers only after real repetition; promote into the shared test kit only when a second suite needs the same helper. +- Do not grow suite-specific “build the whole world” APIs in the shared kit. + +This stays aligned with ADR-0006: factories may widen partial write payloads and fill defaults; they do not redefine row or DAL contracts. + +## Alternatives Considered + +### Alternative 1: Always-on smart factories that scaffold graphs + +- **Pros**: Shortest test setup; one call builds a “ready” world. +- **Cons**: Hides scenario data; easy to pass for the wrong reason; hard to see what a test asserts. +- **Why not**: Tests must declare behavior. Defaults are ergonomics, not fixtures. + +### Alternative 2: No shared defaults — every test passes full insert rows + +- **Pros**: Maximum explicitness; zero magic. +- **Cons**: Noisy boilerplate for ids/labels/flags that rarely matter to the assertion. +- **Why not**: Opt-in allowlisted defaults keep noise down without hiding scenario data. + +### Alternative 3: Defaults baked into every primitive call + +- **Pros**: Slightly shorter call sites. +- **Cons**: Forces defaults even when a test wants raw inserts; harder to see the boundary between persistence and convenience. +- **Why not**: Opt-in composition keeps primitives sharp and defaults obvious at the call site. + +## Consequences + +### Positive + +- Factories stay readable, composable, and reviewable; scenario intent stays in the test. +- Same overwrite semantics everywhere (`undefined` fill, `null` preserved, allowlist only). +- New factories have a clear checklist instead of reinventing “helpful” behavior. +- Stays aligned with ADR-0006’s insert/row types. + +### Negative + +- Callers must compose `withXDefaults` explicitly. +- Suites carry more scenario data in the test file than a mega-helper would. + +### Risks + +- **Defaults creep** — mitigate by reviewing allowlists against “could this change the branch we hit?” and rejecting behavioral defaults in review. +- **Scenario helpers leaking into the shared kit** — mitigate by keeping suite-specific seeds file-local until a second suite needs them. diff --git a/docs/adr/README.md b/docs/adr/README.md index 250066a4a4..839884caed 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -12,6 +12,8 @@ Use the `/adr` skill in Claude Code to record new ADRs or query past decisions. | [ADR-0003](./0003-deps-bq-table-selection.md) | Use DependencyGraphEdgesLatest for deps ingestion; defer DependenciesLatest until NUGET or GO needed | accepted | 2026-05-29 | | [ADR-0004](./0004-go-nuget-transitive-dependent-counts.md) | Compute GO/NUGET transitive dependent counts via exact reverse closure (over HLL approximation) | accepted | 2026-06-23 | | [ADR-0005](./0005-pypi-downloads-bigquery-merge-scoping.md) | PyPI downloads via BigQuery bulk export, scoped in the Postgres merge | accepted | 2026-07-01 | +| [ADR-0006](./0006-database-schema-types-as-source-of-truth.md) | Database schema types as the source of truth | accepted | 2026-07-09 | +| [ADR-0007](./0007-test-factory-primitives-and-defaults.md) | Test factory primitives and defaults | accepted | 2026-07-10 | ## Why ADRs? diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e6565a5487..b85c06984c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2265,6 +2265,9 @@ importers: specifier: ^13.7.0 version: 13.11.0 devDependencies: + '@crowd/test-kit': + specifier: workspace:* + version: link:../test-kit '@types/node': specifier: ^20.8.2 version: 20.12.7 @@ -2274,6 +2277,9 @@ importers: typescript: specifier: ^5.6.3 version: 5.6.3 + vitest: + specifier: 4.1.7 + version: 4.1.7(@types/node@20.12.7)(vite@6.3.5(@types/node@20.12.7)(jiti@2.4.2)(terser@5.43.1)(yaml@2.7.0)) services/libs/database: dependencies: @@ -2597,9 +2603,21 @@ importers: '@crowd/common': specifier: workspace:* version: link:../common + '@crowd/data-access-layer': + specifier: workspace:* + version: link:../data-access-layer '@crowd/database': specifier: workspace:* version: link:../database + '@crowd/integrations': + specifier: workspace:* + version: link:../integrations + '@crowd/types': + specifier: workspace:* + version: link:../types + '@faker-js/faker': + specifier: ^9.9.0 + version: 9.9.0 pg-promise: specifier: ^11.4.3 version: 11.6.0 @@ -3638,6 +3656,10 @@ packages: '@exodus/schemasafe@1.3.0': resolution: {integrity: sha512-5Aap/GaRupgNx/feGBwLLTVv8OQFfv3pq2lPRzPg9R+IOBnDgghTGW7l7EuVXOvg5cc/xSAlRW8rBrjIC3Nvqw==} + '@faker-js/faker@9.9.0': + resolution: {integrity: sha512-OEl393iCOoo/z8bMezRlJu+GlRGlsKbUAN7jKB6LhnKoqKve5DXRpalbItIIcwnCjs1k/FOPjFzcA6Qn+H+YbA==} + engines: {node: '>=18.0.0', npm: '>=9.0.0'} + '@fastify/busboy@2.1.1': resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==} engines: {node: '>=14'} @@ -10609,8 +10631,8 @@ snapshots: dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sso-oidc': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -10804,11 +10826,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sso-oidc@3.572.0(@aws-sdk/client-sts@3.572.0)': + '@aws-sdk/client-sso-oidc@3.572.0': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -10847,7 +10869,6 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: - - '@aws-sdk/client-sts' - aws-crt '@aws-sdk/client-sso@3.556.0': @@ -11023,11 +11044,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sts@3.572.0': + '@aws-sdk/client-sts@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) + '@aws-sdk/client-sso-oidc': 3.572.0 '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -11066,6 +11087,7 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' - aws-crt '@aws-sdk/client-sts@3.985.0': @@ -11231,7 +11253,7 @@ snapshots: '@aws-sdk/credential-provider-ini@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/credential-provider-env': 3.568.0 '@aws-sdk/credential-provider-process': 3.572.0 '@aws-sdk/credential-provider-sso': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) @@ -11408,7 +11430,7 @@ snapshots: '@aws-sdk/credential-provider-web-identity@3.568.0(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/types': 2.12.0 @@ -11720,7 +11742,7 @@ snapshots: '@aws-sdk/token-providers@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': dependencies: - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) + '@aws-sdk/client-sso-oidc': 3.572.0 '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/shared-ini-file-loader': 2.4.0 @@ -12427,6 +12449,8 @@ snapshots: '@exodus/schemasafe@1.3.0': {} + '@faker-js/faker@9.9.0': {} + '@fastify/busboy@2.1.1': {} '@gitbeaker/core@43.5.0': @@ -19204,7 +19228,7 @@ snapshots: dependencies: '@types/debug': 4.1.12 '@types/validator': 13.11.9 - debug: 4.4.0(supports-color@5.5.0) + debug: 4.4.3 dottie: 2.0.6 inflection: 1.13.4 lodash: 4.17.21 diff --git a/services/apps/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/members_enrichment_worker/src/activities/enrichment.ts index 726375f659..9c54ade13b 100644 --- a/services/apps/members_enrichment_worker/src/activities/enrichment.ts +++ b/services/apps/members_enrichment_worker/src/activities/enrichment.ts @@ -36,9 +36,9 @@ import { } from '@crowd/data-access-layer/src/old/apps/members_enrichment_worker' import OrganizationMergeSuggestionsRepository from '@crowd/data-access-layer/src/old/apps/merge_suggestions_worker/organizationMergeSuggestions.repo' import { - addOrgIdentity, findOrCreateOrganization, findOrgByVerifiedIdentity, + insertOrganizationIdentities, } from '@crowd/data-access-layer/src/organizations' import { dbStoreQx, pgpQx } from '@crowd/data-access-layer/src/queryExecutor' import { refreshMaterializedView } from '@crowd/data-access-layer/src/utils' @@ -518,17 +518,23 @@ export async function updateMemberUsingSquashedPayload( const mergeSuggestions = [] const suggestedOwnerIds = new Set() + const identitiesToInsert = identityOwners + .filter((identityOwner) => identityOwner.organizationId !== orgId) + .map((identityOwner) => ({ + organizationId: orgId, + platform: identityOwner.identity.platform, + value: identityOwner.identity.value, + type: identityOwner.identity.type, + verified: false, + source: orgSource, + })) + + if (identitiesToInsert.length > 0) { + await insertOrganizationIdentities(qx, identitiesToInsert, false) + } + for (const identityOwner of identityOwners) { if (identityOwner.organizationId !== orgId) { - await addOrgIdentity(qx, { - organizationId: orgId, - platform: identityOwner.identity.platform, - value: identityOwner.identity.value, - type: identityOwner.identity.type, - verified: false, - source: orgSource, - }) - const noMergeIds = await mergeSuggestionsRepo.findNoMergeIds( identityOwner.organizationId, ) diff --git a/services/libs/data-access-layer/package.json b/services/libs/data-access-layer/package.json index bb5d64a311..d14b776913 100644 --- a/services/libs/data-access-layer/package.json +++ b/services/libs/data-access-layer/package.json @@ -33,8 +33,10 @@ "validator": "^13.7.0" }, "devDependencies": { + "@crowd/test-kit": "workspace:*", "@types/node": "^20.8.2", "sequelize": "6.37.8", - "typescript": "^5.6.3" + "typescript": "^5.6.3", + "vitest": "4.1.7" } } diff --git a/services/libs/data-access-layer/src/activities/sql.ts b/services/libs/data-access-layer/src/activities/sql.ts index 54cbac19e8..0887cb7e7b 100644 --- a/services/libs/data-access-layer/src/activities/sql.ts +++ b/services/libs/data-access-layer/src/activities/sql.ts @@ -9,7 +9,7 @@ import { Counter, TinybirdClient, } from '@crowd/database' -import { ActivityDisplayService } from '@crowd/integrations' +import { ActivityDisplayService } from '@crowd/integrations/src/integrations/activityDisplayService' import { ActivityTypeSettings, ITimeseriesDatapoint, PageData } from '@crowd/types' import { getLatestMemberActivityRelations } from '../activityRelations' diff --git a/services/libs/data-access-layer/src/index.ts b/services/libs/data-access-layer/src/index.ts index 9ede0d95cd..3037d24c91 100644 --- a/services/libs/data-access-layer/src/index.ts +++ b/services/libs/data-access-layer/src/index.ts @@ -6,10 +6,12 @@ export * from './dashboards' export * from './members' export * from './organizations' export * from './member-organization-affiliation' +export * from './member_segment_affiliations' export * from './prompt-history' export * from './queryExecutor' export * from './repositories' export * from './security_insights' +export * from './segments' export * from './systemSettings' export * from './integrations' export * from './auditLogs' @@ -21,9 +23,6 @@ export * from './osspckgs/maintainers' export * from './osspckgs/packages' export * from './osspckgs/repos' export * from './osspckgs/stewardships' -export * from './osspckgs/types' export * from './osspckgs/versions' export * from './osspckgs/nuget' -export * from './osspckgs/downloads' -export * from './osspckgs/rubygems' export * from './osspckgs/api' diff --git a/services/libs/data-access-layer/src/member-organization-affiliation/index.ts b/services/libs/data-access-layer/src/member-organization-affiliation/index.ts index 0e3f3115bb..0ccfcd8198 100644 --- a/services/libs/data-access-layer/src/member-organization-affiliation/index.ts +++ b/services/libs/data-access-layer/src/member-organization-affiliation/index.ts @@ -7,6 +7,7 @@ import { IChangeAffiliationOverrideData, IMemberOrganization, IMemberOrganizationAffiliationOverride, + type MemberOrganizationAffiliationOverrideDbRow, } from '@crowd/types' import { findMemberAffiliations } from '../member_segment_affiliations' @@ -31,7 +32,7 @@ const isMemberOrganizationWithOverrides = ( row: AffiliationItem, ): row is MemberOrganizationWithOverrides => !isManualAffiliation(row) -async function prepareMemberOrganizationAffiliationTimeline( +export async function prepareMemberOrganizationAffiliationTimeline( qx: QueryExecutor, memberId: string, ): Promise { @@ -522,9 +523,20 @@ export async function refreshMemberOrganizationAffiliations(qx: QueryExecutor, m export async function changeMemberOrganizationAffiliationOverrides( qx: QueryExecutor, data: IChangeAffiliationOverrideData[], -): Promise { + returnRows: true, +): Promise +export async function changeMemberOrganizationAffiliationOverrides( + qx: QueryExecutor, + data: IChangeAffiliationOverrideData[], + returnRows?: false, +): Promise +export async function changeMemberOrganizationAffiliationOverrides( + qx: QueryExecutor, + data: IChangeAffiliationOverrideData[], + returnRows = false, +): Promise { if (!Array.isArray(data) || data.length === 0) { - return + return returnRows ? [] : undefined } const rows: IMemberOrganizationAffiliationOverride[] = [] @@ -548,7 +560,7 @@ export async function changeMemberOrganizationAffiliationOverrides( } if (rows.length === 0) { - return + return returnRows ? [] : undefined } const valuesSql = rows @@ -577,8 +589,7 @@ export async function changeMemberOrganizationAffiliationOverrides( {} as Record, ) - await qx.result( - ` + const query = ` INSERT INTO "memberOrganizationAffiliationOverrides" ( id, "memberId", @@ -590,10 +601,15 @@ export async function changeMemberOrganizationAffiliationOverrides( ON CONFLICT ("memberId", "memberOrganizationId") DO UPDATE SET "allowAffiliation" = COALESCE(EXCLUDED."allowAffiliation", "memberOrganizationAffiliationOverrides"."allowAffiliation"), - "isPrimaryWorkExperience" = COALESCE(EXCLUDED."isPrimaryWorkExperience", "memberOrganizationAffiliationOverrides"."isPrimaryWorkExperience"); - `, - params, - ) + "isPrimaryWorkExperience" = COALESCE(EXCLUDED."isPrimaryWorkExperience", "memberOrganizationAffiliationOverrides"."isPrimaryWorkExperience") + ${returnRows ? 'RETURNING *' : ''} + ` + + if (returnRows) { + return qx.select(query, params) + } + + await qx.result(query, params) } export async function findMemberAffiliationOverrides( diff --git a/services/libs/data-access-layer/src/member_segment_affiliations/index.ts b/services/libs/data-access-layer/src/member_segment_affiliations/index.ts index 255b6077c4..97af82a1cf 100644 --- a/services/libs/data-access-layer/src/member_segment_affiliations/index.ts +++ b/services/libs/data-access-layer/src/member_segment_affiliations/index.ts @@ -1,6 +1,9 @@ -import { v4 as uuid } from 'uuid' - -import { IMemberAffiliation } from '@crowd/types' +import { generateUUIDv1 } from '@crowd/common' +import { + type IMemberAffiliation, + type MemberSegmentAffiliationDbInsert, + type MemberSegmentAffiliationDbRow, +} from '@crowd/types' import { IManualAffiliationData } from '../old/apps/data_sink_worker/repo/memberAffiliation.data' import { QueryExecutor } from '../queryExecutor' @@ -78,21 +81,75 @@ export async function findMemberAffiliations( ) } +export async function insertMemberSegmentAffiliationRows( + qx: QueryExecutor, + affiliations: MemberSegmentAffiliationDbInsert[], + failOnConflict: boolean, + returnRows: true, +): Promise +export async function insertMemberSegmentAffiliationRows( + qx: QueryExecutor, + affiliations: MemberSegmentAffiliationDbInsert[], + failOnConflict?: boolean, + returnRows?: false, +): Promise +export async function insertMemberSegmentAffiliationRows( + qx: QueryExecutor, + affiliations: MemberSegmentAffiliationDbInsert[], + failOnConflict = false, + returnRows = false, +): Promise { + if (affiliations.length === 0) { + return returnRows ? [] : 0 + } + + const query = prepareBulkInsert( + 'memberSegmentAffiliations', + [ + 'id', + 'memberId', + 'segmentId', + 'organizationId', + 'dateStart', + 'dateEnd', + 'verified', + 'verifiedBy', + ], + affiliations.map((a) => ({ + id: a.id ?? generateUUIDv1(), + memberId: a.memberId, + segmentId: a.segmentId, + organizationId: a.organizationId ?? null, + dateStart: a.dateStart ?? null, + dateEnd: a.dateEnd ?? null, + verified: a.verified ?? false, + verifiedBy: a.verifiedBy ?? null, + })), + failOnConflict ? undefined : 'DO NOTHING', + returnRows, + ) + + if (returnRows) { + return qx.select(query) + } + + return qx.result(query) +} + +/** @deprecated Prefer `insertMemberSegmentAffiliationRows` with full insert payloads. */ // eslint-disable-next-line @typescript-eslint/no-explicit-any export async function insertMemberAffiliations(qx: QueryExecutor, memberId: string, data: any[]) { - return qx.result( - prepareBulkInsert( - 'memberSegmentAffiliations', - ['id', 'memberId', 'segmentId', 'organizationId', 'dateStart', 'dateEnd'], - data.map((item) => ({ - id: uuid(), - memberId, - segmentId: item.segmentId, - organizationId: item.organizationId, - dateStart: item.dateStart || null, - dateEnd: item.dateEnd || null, - })), - ), + return insertMemberSegmentAffiliationRows( + qx, + data.map((item) => ({ + memberId, + segmentId: item.segmentId, + organizationId: item.organizationId, + dateStart: item.dateStart || null, + dateEnd: item.dateEnd || null, + })), + false, + false, ) } diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 767db22dcc..9f9ede74d7 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -15,6 +15,8 @@ import { ALL_PLATFORM_TYPES, IMemberContribution, MemberAttributeType, + MemberDbInsert, + MemberDbRow, MemberRow, PageData, SegmentType, @@ -75,15 +77,8 @@ export enum MemberField { UPDATED_BY_ID = 'updatedById', } -export interface MemberCreateInput { - id?: string - displayName: string - joinedAt: string - attributes: Record - reach: Partial> - manuallyCreated?: boolean - contributions?: IMemberContribution[] -} +/** @deprecated Prefer `MemberDbInsert` from `@crowd/types`. */ +export type MemberCreateInput = MemberDbInsert export interface MemberUpdateInput { joinedAt?: string @@ -132,13 +127,19 @@ export const MEMBER_INSERT_COLUMNS = [ 'attributes', 'contributions', 'createdAt', + 'createdById', 'displayName', + 'enrichedBy', 'id', + 'importHash', 'joinedAt', + 'manuallyChangedFields', 'manuallyCreated', 'reach', + 'score', 'tenantId', 'updatedAt', + 'updatedById', ] const QUERY_FILTER_COLUMN_MAP: Map = new Map([ @@ -752,7 +753,7 @@ export async function updateMember( return qx.selectOneOrNone(`${query} ${condition} returning *`) } -export async function createMember(qx: QueryExecutor, data: MemberCreateInput): Promise { +export async function createMember(qx: QueryExecutor, data: MemberDbInsert): Promise { const id = data.id ?? generateUUIDv1() const ts = new Date() const dbInstance = getDbInstance() diff --git a/services/libs/data-access-layer/src/members/identities.ts b/services/libs/data-access-layer/src/members/identities.ts index 10a2280837..d7d47f4939 100644 --- a/services/libs/data-access-layer/src/members/identities.ts +++ b/services/libs/data-access-layer/src/members/identities.ts @@ -1,8 +1,9 @@ -import { DEFAULT_TENANT_ID } from '@crowd/common' +import { DEFAULT_TENANT_ID, generateUUIDv1 } from '@crowd/common' import { IMemberIdentity, + MemberIdentityDbInsert, + MemberIdentityDbRow, MemberIdentityType, - NewMemberIdentity, PlatformType, UpdateMemberIdentity, } from '@crowd/types' @@ -233,27 +234,32 @@ export async function findMemberIdByVerifiedIdentity( return result?.memberId ?? null } -export async function insertManyMemberIdentities( +export async function insertMemberIdentities( qx: QueryExecutor, - identities: NewMemberIdentity[], + identities: MemberIdentityDbInsert[], failOnConflict: boolean, returnRows: true, -): Promise -export async function insertManyMemberIdentities( +): Promise +export async function insertMemberIdentities( qx: QueryExecutor, - identities: NewMemberIdentity[], + identities: MemberIdentityDbInsert[], failOnConflict?: boolean, returnRows?: false, ): Promise -export async function insertManyMemberIdentities( +export async function insertMemberIdentities( qx: QueryExecutor, - identities: NewMemberIdentity[], + identities: MemberIdentityDbInsert[], failOnConflict = false, returnRows = false, -): Promise { +): Promise { + if (identities.length === 0) { + return returnRows ? [] : 0 + } + const query = prepareBulkInsert( 'memberIdentities', [ + 'id', 'memberId', 'tenantId', 'integrationId', @@ -265,12 +271,19 @@ export async function insertManyMemberIdentities( 'verified', 'verifiedBy', ], - identities.map((i) => { - return { - tenantId: DEFAULT_TENANT_ID, - ...i, - } - }), + identities.map((i) => ({ + id: i.id ?? generateUUIDv1(), + memberId: i.memberId, + tenantId: DEFAULT_TENANT_ID, + integrationId: i.integrationId ?? null, + platform: i.platform, + source: i.source ?? null, + sourceId: i.sourceId ?? null, + value: i.value, + type: i.type, + verified: i.verified ?? true, + verifiedBy: i.verifiedBy ?? null, + })), failOnConflict ? undefined : 'DO NOTHING', returnRows, ) @@ -284,28 +297,28 @@ export async function insertManyMemberIdentities( export async function createMemberIdentity( qx: QueryExecutor, - i: NewMemberIdentity, + i: MemberIdentityDbInsert, failOnConflict: boolean, returnRows: true, -): Promise +): Promise export async function createMemberIdentity( qx: QueryExecutor, - i: NewMemberIdentity, + i: MemberIdentityDbInsert, failOnConflict?: boolean, returnRows?: false, ): Promise export async function createMemberIdentity( qx: QueryExecutor, - i: NewMemberIdentity, + i: MemberIdentityDbInsert, failOnConflict = false, returnRows = false, -): Promise { +): Promise { if (returnRows) { - const rows = await insertManyMemberIdentities(qx, [i], failOnConflict, true) + const rows = await insertMemberIdentities(qx, [i], failOnConflict, true) return rows[0] } - await insertManyMemberIdentities(qx, [i], failOnConflict) + await insertMemberIdentities(qx, [i], failOnConflict) } export async function moveToNewMember( diff --git a/services/libs/data-access-layer/src/members/organizations.ts b/services/libs/data-access-layer/src/members/organizations.ts index 594ed21654..8d846b9620 100644 --- a/services/libs/data-access-layer/src/members/organizations.ts +++ b/services/libs/data-access-layer/src/members/organizations.ts @@ -1,7 +1,10 @@ +import { generateUUIDv1 } from '@crowd/common' import { IMemberOrganization, IMemberOrganizationAffiliationOverride, IMemberRoleWithOrganization, + MemberOrganizationDbInsert, + MemberOrganizationDbRow, MemberOrganizationUpdate, OrganizationSource, } from '@crowd/types' @@ -14,6 +17,7 @@ import { import { deleteMemberSegmentAffiliations } from '../member_segment_affiliations' import { EntityType } from '../old/apps/script_executor_worker/types' import { QueryExecutor } from '../queryExecutor' +import { prepareBulkInsert } from '../utils' import { EmailDomainMemberOrganizationActivityDate } from './types' @@ -293,53 +297,93 @@ export async function fetchManyOrganizationAffiliationPolicies( ) } +export async function insertMemberOrganizations( + qx: QueryExecutor, + organizations: MemberOrganizationDbInsert[], + failOnConflict: boolean, + returnRows: true, +): Promise +export async function insertMemberOrganizations( + qx: QueryExecutor, + organizations: MemberOrganizationDbInsert[], + failOnConflict?: boolean, + returnRows?: false, +): Promise +export async function insertMemberOrganizations( + qx: QueryExecutor, + organizations: MemberOrganizationDbInsert[], + failOnConflict = false, + returnRows = false, +): Promise { + const ts = new Date() + + if (organizations.length === 0) { + return returnRows ? [] : 0 + } + + const query = prepareBulkInsert( + 'memberOrganizations', + [ + 'id', + 'memberId', + 'organizationId', + 'dateStart', + 'dateEnd', + 'title', + 'source', + 'verified', + 'verifiedBy', + 'createdAt', + 'updatedAt', + ], + organizations.map((o) => ({ + id: o.id ?? generateUUIDv1(), + memberId: o.memberId, + organizationId: o.organizationId, + dateStart: o.dateStart ?? null, + dateEnd: o.dateEnd ?? null, + title: o.title ?? null, + source: o.source ?? null, + verified: o.verified ?? false, + verifiedBy: o.verifiedBy ?? null, + createdAt: ts, + updatedAt: ts, + })), + failOnConflict ? undefined : 'DO NOTHING', + returnRows, + ) + + if (returnRows) { + return qx.select(query) + } + + return qx.result(query) +} + export async function createMemberOrganization( qx: QueryExecutor, memberId: string, data: Partial, ): Promise { - const result = await qx.selectOneOrNone( - ` - INSERT INTO "memberOrganizations"( - "memberId", - "organizationId", - "dateStart", - "dateEnd", - "title", - "source", - "verified", - "verifiedBy", - "createdAt", - "updatedAt" - ) - VALUES( - $(memberId), - $(organizationId), - $(dateStart), - $(dateEnd), - $(title), - $(source), - $(verified), - $(verifiedBy), - now(), - now() - ) - ON CONFLICT DO NOTHING - RETURNING id - `, - { - memberId, - organizationId: data.organizationId, - dateStart: data.dateStart ?? null, - dateEnd: data.dateEnd ?? null, - title: data.title ?? null, - source: data.source ?? null, - verified: data.verified ?? false, - verifiedBy: data.verifiedBy ?? null, - }, + const rows = await insertMemberOrganizations( + qx, + [ + { + memberId, + organizationId: data.organizationId, + dateStart: data.dateStart != null ? toIsoString(data.dateStart) : null, + dateEnd: data.dateEnd != null ? toIsoString(data.dateEnd) : null, + title: data.title, + source: data.source, + verified: data.verified, + verifiedBy: data.verifiedBy, + }, + ], + false, + true, ) - return result?.id + return rows[0]?.id } export async function createOrUpdateMemberOrganizations( diff --git a/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/member.repo.ts b/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/member.repo.ts index 72e2ce285d..ae2bc305b5 100644 --- a/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/member.repo.ts +++ b/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/member.repo.ts @@ -3,10 +3,7 @@ import { DbColumnSet, DbStore, RepositoryBase } from '@crowd/database' import { Logger } from '@crowd/logging' import { IMemberIdentity, MemberIdentityType } from '@crowd/types' -import { - deleteManyMemberIdentities, - insertManyMemberIdentities, -} from '../../../../members/identities' +import { deleteManyMemberIdentities, insertMemberIdentities } from '../../../../members/identities' import { PgPromiseQueryExecutor } from '../../../../queryExecutor' import { IDbMember, getInsertMemberColumnSet, getSelectMemberColumnSet } from './member.data' @@ -101,11 +98,7 @@ export default class MemberRepository extends RepositoryBase { } }) - return insertManyMemberIdentities( - new PgPromiseQueryExecutor(this.db()), - objects, - failOnConflict, - ) + return insertMemberIdentities(new PgPromiseQueryExecutor(this.db()), objects, failOnConflict) } public async addToSegments(memberId: string, segmentIds: string[]): Promise { diff --git a/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/settings.repo.ts b/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/settings.repo.ts index 0c37a4e96e..ab60083eaa 100644 --- a/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/settings.repo.ts +++ b/services/libs/data-access-layer/src/old/apps/data_sink_worker/repo/settings.repo.ts @@ -1,6 +1,6 @@ import { DEFAULT_TENANT_ID } from '@crowd/common' import { DbStore, RepositoryBase } from '@crowd/database' -import { DEFAULT_ACTIVITY_TYPE_SETTINGS } from '@crowd/integrations' +import { DEFAULT_ACTIVITY_TYPE_SETTINGS } from '@crowd/integrations/src/integrations/activityTypes' import { Logger } from '@crowd/logging' import { PlatformType } from '@crowd/types' diff --git a/services/libs/data-access-layer/src/organizations/base.ts b/services/libs/data-access-layer/src/organizations/base.ts index 3323ba09fc..cd766a7878 100644 --- a/services/libs/data-access-layer/src/organizations/base.ts +++ b/services/libs/data-access-layer/src/organizations/base.ts @@ -12,16 +12,18 @@ import { IOrganizationIdentity, IQueryTimeseriesParams, ITimeseriesDatapoint, + OrganizationDbInsert, + OrganizationDbRow, OrganizationIdentityType, } from '@crowd/types' import { QueryExecutor } from '../queryExecutor' import { findLfSegmentByName } from '../segments' -import { QueryOptions, QueryResult, queryTable, queryTableById } from '../utils' +import { QueryOptions, QueryResult, prepareBulkInsert, queryTable, queryTableById } from '../utils' import { prepareSelectColumns } from '../utils' import { findOrgAttributes, markOrgAttributeDefault, upsertOrgAttributes } from './attributes' -import { addOrgIdentity, upsertOrgIdentities } from './identities' +import { insertOrganizationIdentities, upsertOrgIdentities } from './identities' import { IDbOrganization, IDbOrganizationInput } from './types' import { prepareOrganizationData } from './utils' @@ -363,6 +365,95 @@ export async function insertOrganization( return id } +export async function insertOrganizations( + qx: QueryExecutor, + organizations: OrganizationDbInsert[], + failOnConflict: boolean, + returnRows: true, +): Promise +export async function insertOrganizations( + qx: QueryExecutor, + organizations: OrganizationDbInsert[], + failOnConflict?: boolean, + returnRows?: false, +): Promise +export async function insertOrganizations( + qx: QueryExecutor, + organizations: OrganizationDbInsert[], + failOnConflict = false, + returnRows = false, +): Promise { + if (organizations.length === 0) { + return returnRows ? [] : 0 + } + + const ts = new Date() + + const query = prepareBulkInsert( + 'organizations', + [ + 'id', + 'displayName', + 'description', + 'logo', + 'tags', + 'employees', + 'revenueRange', + 'importHash', + 'location', + 'isTeamOrganization', + 'isAffiliationBlocked', + 'type', + 'size', + 'headline', + 'industry', + 'founded', + 'employeeChurnRate', + 'employeeGrowthRate', + 'manuallyCreated', + 'createdById', + 'updatedById', + 'tenantId', + 'createdAt', + 'updatedAt', + ], + organizations.map((o) => ({ + id: o.id ?? generateUUIDv1(), + displayName: o.displayName ?? null, + description: o.description ?? null, + logo: o.logo ?? null, + tags: o.tags ?? null, + employees: o.employees ?? null, + revenueRange: o.revenueRange ?? null, + importHash: o.importHash ?? null, + location: o.location ?? null, + isTeamOrganization: o.isTeamOrganization ?? false, + isAffiliationBlocked: o.isAffiliationBlocked ?? false, + type: o.type ?? null, + size: o.size ?? null, + headline: o.headline ?? null, + industry: o.industry ?? null, + founded: o.founded ?? null, + employeeChurnRate: o.employeeChurnRate ?? null, + employeeGrowthRate: o.employeeGrowthRate ?? null, + manuallyCreated: o.manuallyCreated ?? false, + createdById: o.createdById ?? null, + updatedById: o.updatedById ?? null, + tenantId: DEFAULT_TENANT_ID, + createdAt: ts, + updatedAt: ts, + })), + failOnConflict ? undefined : 'DO NOTHING', + returnRows, + ) + + if (returnRows) { + return qx.select(query) + } + + return qx.result(query) +} + export async function updateOrganization( qe: QueryExecutor, organizationId: string, @@ -628,22 +719,25 @@ export async function findOrCreateOrganization( } // create identities - for (const i of data.identities) { - // add the identity + if (data.identities.length > 0) { await logExecutionTimeV2( - async () => - addOrgIdentity(qe, { - organizationId: id, - platform: i.platform, - type: i.type, - value: i.value, - verified: i.verified, - sourceId: i.sourceId, - integrationId, - source: i.source, - }), + () => + insertOrganizationIdentities( + qe, + data.identities.map((i) => ({ + organizationId: id, + platform: i.platform, + type: i.type, + value: i.value, + verified: i.verified, + sourceId: i.sourceId, + integrationId, + source: i.source, + })), + false, + ), log, - 'organizationService -> findOrCreateOrganization -> addOrgIdentity', + 'organizationService -> findOrCreateOrganization -> insertOrganizationIdentities', ) } } diff --git a/services/libs/data-access-layer/src/organizations/identities.ts b/services/libs/data-access-layer/src/organizations/identities.ts index 14aa5956a3..13381bea6d 100644 --- a/services/libs/data-access-layer/src/organizations/identities.ts +++ b/services/libs/data-access-layer/src/organizations/identities.ts @@ -1,12 +1,13 @@ import { DEFAULT_TENANT_ID } from '@crowd/common' import { IOrganizationIdentity, - NewOrganizationIdentity, + type OrganizationIdentityDbInsert, + type OrganizationIdentityDbRow, OrganizationIdentityType, } from '@crowd/types' import { QueryExecutor } from '../queryExecutor' -import { QueryOptions, QueryResult, queryTable } from '../utils' +import { QueryOptions, QueryResult, prepareBulkInsert, queryTable } from '../utils' import { IDbOrgIdentityUpdateInput } from './types' @@ -71,26 +72,61 @@ export async function updateOrgIdentityVerifiedFlag( ) } -export async function addOrgIdentity(qx: QueryExecutor, identity: NewOrganizationIdentity) { - return qx.result( - ` - INSERT INTO "organizationIdentities" ( - "organizationId", - "source", - platform, - value, - type, - verified, - "sourceId", - "tenantId", - "integrationId", - "createdAt" - ) - VALUES ($(organizationId), $(source), $(platform), $(value), $(type), $(verified), $(sourceId), $(tenantId), $(integrationId), NOW()) - ON CONFLICT DO NOTHING; - `, - { tenantId: DEFAULT_TENANT_ID, ...identity }, +export async function insertOrganizationIdentities( + qx: QueryExecutor, + identities: OrganizationIdentityDbInsert[], + failOnConflict: boolean, + returnRows: true, +): Promise +export async function insertOrganizationIdentities( + qx: QueryExecutor, + identities: OrganizationIdentityDbInsert[], + failOnConflict?: boolean, + returnRows?: false, +): Promise +export async function insertOrganizationIdentities( + qx: QueryExecutor, + identities: OrganizationIdentityDbInsert[], + failOnConflict = false, + returnRows = false, +): Promise { + if (identities.length === 0) { + return returnRows ? [] : 0 + } + + const query = prepareBulkInsert( + 'organizationIdentities', + [ + 'organizationId', + 'platform', + 'value', + 'type', + 'verified', + 'source', + 'sourceId', + 'tenantId', + 'integrationId', + ], + identities.map((i) => ({ + organizationId: i.organizationId, + platform: i.platform, + value: i.value, + type: i.type, + verified: i.verified, + source: i.source ?? null, + sourceId: i.sourceId ?? null, + tenantId: DEFAULT_TENANT_ID, + integrationId: i.integrationId ?? null, + })), + failOnConflict ? undefined : 'DO NOTHING', + returnRows, ) + + if (returnRows) { + return qx.select(query) + } + + return qx.result(query) } export async function upsertOrgIdentities( @@ -100,47 +136,41 @@ export async function upsertOrgIdentities( integrationId?: string, ): Promise { const existingIdentities = await fetchOrgIdentities(qe, organizationId) - const toCreate = [] - const toUpdate = [] + const toCreate: OrganizationIdentityDbInsert[] = [] + const toUpdate: Partial[] = [] for (const i of identities) { const existing = existingIdentities.find( (ei) => ei.value === i.value && ei.platform === i.platform && ei.type === i.type, ) if (!existing) { - toCreate.push(i) - } else if (existing && existing.verified !== i.verified) { - toUpdate.push(i) - } - } - - if (toCreate.length > 0) { - for (const i of toCreate) { - // add the identity - await addOrgIdentity(qe, { + toCreate.push({ organizationId, platform: i.platform, type: i.type, value: i.value, verified: i.verified, - source: i.source, - sourceId: i.sourceId, - integrationId, + source: i.source ?? null, + sourceId: i.sourceId ?? null, + integrationId: integrationId ?? null, }) + } else if (existing.verified !== i.verified) { + toUpdate.push(i) } } - if (toUpdate.length > 0) { - for (const i of toUpdate) { - // update the identity - await updateOrgIdentityVerifiedFlag(qe, { - organizationId, - platform: i.platform, - type: i.type, - value: i.value, - verified: i.verified, - }) - } + if (toCreate.length > 0) { + await insertOrganizationIdentities(qe, toCreate, false) + } + + for (const i of toUpdate) { + await updateOrgIdentityVerifiedFlag(qe, { + organizationId, + platform: i.platform, + type: i.type, + value: i.value, + verified: i.verified, + }) } } diff --git a/services/libs/data-access-layer/src/segments/index.ts b/services/libs/data-access-layer/src/segments/index.ts index 19498a39a5..fe862309e9 100644 --- a/services/libs/data-access-layer/src/segments/index.ts +++ b/services/libs/data-access-layer/src/segments/index.ts @@ -1,18 +1,22 @@ import lodash from 'lodash' import cloneDeep from 'lodash.clonedeep' -import { DEFAULT_TENANT_ID } from '@crowd/common' -import { DEFAULT_ACTIVITY_TYPE_SETTINGS } from '@crowd/integrations' +import { DEFAULT_TENANT_ID, generateUUIDv1 } from '@crowd/common' +import { DEFAULT_ACTIVITY_TYPE_SETTINGS } from '@crowd/integrations/src/integrations/activityTypes' import { ActivityTypeSettings, MergeActionState, MergeActionType, PlatformType, SegmentData, + SegmentDbInsert, + SegmentDbRow, SegmentRawData, + SegmentStatus, } from '@crowd/types' import { QueryExecutor } from '../queryExecutor' +import { prepareBulkInsert } from '../utils' export async function findProjectGroupByName( qx: QueryExecutor, @@ -164,6 +168,86 @@ export function isSegmentSubproject(segment: SegmentData | SegmentRawData): bool return segment.slug != null && segment.parentSlug != null && segment.grandparentSlug != null } +export async function insertSegments( + qx: QueryExecutor, + segments: SegmentDbInsert[], + failOnConflict: boolean, + returnRows: true, +): Promise +export async function insertSegments( + qx: QueryExecutor, + segments: SegmentDbInsert[], + failOnConflict?: boolean, + returnRows?: false, +): Promise +export async function insertSegments( + qx: QueryExecutor, + segments: SegmentDbInsert[], + failOnConflict = false, + returnRows = false, +): Promise { + const ts = new Date() + + const query = prepareBulkInsert( + 'segments', + [ + 'id', + 'slug', + 'name', + 'url', + 'status', + 'isLF', + 'parentName', + 'grandparentName', + 'parentSlug', + 'grandparentSlug', + 'description', + 'sourceId', + 'sourceParentId', + 'customActivityTypes', + 'activityChannels', + 'parentId', + 'grandparentId', + // `type` is GENERATED ALWAYS from parentSlug/grandparentSlug — do not insert + 'maturity', + 'tenantId', + 'createdAt', + 'updatedAt', + ], + segments.map((s) => ({ + id: s.id ?? generateUUIDv1(), + slug: s.slug, + name: s.name ?? null, + url: s.url ?? null, + status: s.status ?? SegmentStatus.ACTIVE, + isLF: s.isLF ?? true, + parentName: s.parentName ?? null, + grandparentName: s.grandparentName ?? null, + parentSlug: s.parentSlug ?? null, + grandparentSlug: s.grandparentSlug ?? null, + description: s.description ?? null, + sourceId: s.sourceId ?? null, + sourceParentId: s.sourceParentId ?? null, + customActivityTypes: s.customActivityTypes ?? {}, + activityChannels: s.activityChannels ?? {}, + parentId: s.parentId ?? null, + grandparentId: s.grandparentId ?? null, + maturity: s.maturity ?? null, + tenantId: DEFAULT_TENANT_ID, + createdAt: ts, + updatedAt: ts, + })), + failOnConflict ? undefined : 'DO NOTHING', + returnRows, + ) + + if (returnRows) { + return qx.select(query) + } + + return qx.result(query) +} + export async function getSegmentSubprojects( qx: QueryExecutor, segmentIds: string[], diff --git a/services/libs/test-kit/package.json b/services/libs/test-kit/package.json index 2ba5fe2ebf..365ca89364 100644 --- a/services/libs/test-kit/package.json +++ b/services/libs/test-kit/package.json @@ -1,8 +1,11 @@ { "name": "@crowd/test-kit", "private": true, + "sideEffects": false, "exports": { - "./fixtures": "./src/fixtures.ts" + "./db": "./src/db/index.ts", + "./factories": "./src/factories/index.ts", + "./faker": "./src/faker.ts" }, "scripts": { "lint": "npx eslint --ext .ts src --max-warnings=0", @@ -12,7 +15,11 @@ }, "dependencies": { "@crowd/common": "workspace:*", + "@crowd/data-access-layer": "workspace:*", "@crowd/database": "workspace:*", + "@crowd/integrations": "workspace:*", + "@crowd/types": "workspace:*", + "@faker-js/faker": "^9.9.0", "pg-promise": "^11.4.3" }, "devDependencies": { diff --git a/services/libs/test-kit/src/db/index.ts b/services/libs/test-kit/src/db/index.ts new file mode 100644 index 0000000000..1c0dbaaf69 --- /dev/null +++ b/services/libs/test-kit/src/db/index.ts @@ -0,0 +1,2 @@ +export { openTestWorkerDatabase, resetTestDatabase, seedTestBaseline } from './postgres' +export { withQx } from './qx' diff --git a/services/libs/test-kit/src/postgres.ts b/services/libs/test-kit/src/db/postgres.ts similarity index 98% rename from services/libs/test-kit/src/postgres.ts rename to services/libs/test-kit/src/db/postgres.ts index c69878b86d..7303ce5434 100644 --- a/services/libs/test-kit/src/postgres.ts +++ b/services/libs/test-kit/src/db/postgres.ts @@ -4,7 +4,12 @@ import { DEFAULT_TENANT_ID, IS_TEST_ENV } from '@crowd/common' import { type DbConnection, type DbInstance, type IDatabaseConfig } from '@crowd/database' import type { QueryExecutor } from '@crowd/database' -import type { TestPostgres } from './types' +type TestPostgres = { + host: string + port: number + user: string + password: string +} /** * Creates an isolated, dynamic test database cloned from the template database diff --git a/services/libs/test-kit/src/fixtures.ts b/services/libs/test-kit/src/db/qx.ts similarity index 100% rename from services/libs/test-kit/src/fixtures.ts rename to services/libs/test-kit/src/db/qx.ts diff --git a/services/libs/test-kit/src/factories/activity-relation.ts b/services/libs/test-kit/src/factories/activity-relation.ts new file mode 100644 index 0000000000..962a5276d6 --- /dev/null +++ b/services/libs/test-kit/src/factories/activity-relation.ts @@ -0,0 +1,172 @@ +import { faker } from '@faker-js/faker' + +import { generateUUIDv1 } from '@crowd/common' +import { createOrUpdateRelations } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import { DEFAULT_ACTIVITY_TYPE_SETTINGS } from '@crowd/integrations/src/integrations/activityTypes' +import { + type ActivityRelationDbInsert, + type MemberIdentityDbRow, + type MemberOrganizationDbRow, + PlatformType, +} from '@crowd/types' + +import { withDefaults } from './defaults' + +const DEFAULT_COUNT = 100 + +interface GenerateActivityRelationsInput { + memberId: string + segmentId: string + identities: MemberIdentityDbRow[] + timestamp: { + from: string + to: string + } + count?: number + memberOrganizations?: MemberOrganizationDbRow[] +} + +export const withActivityRelationDefaults = ( + data: Partial[], +): ActivityRelationDbInsert[] => + withDefaults({ + activityId: () => generateUUIDv1(), + sourceId: () => generateUUIDv1(), + })(data) + +/** Build N activityRelation rows from member context. */ +export function generateActivityRelations( + input: GenerateActivityRelationsInput, +): ActivityRelationDbInsert[] { + const { + memberId, + segmentId, + identities, + timestamp, + memberOrganizations, + count = DEFAULT_COUNT, + } = input + + if (identities.length === 0) { + throw new Error('generateActivityRelations requires at least one identity') + } + + if (count < 1) { + throw new Error('generateActivityRelations count must be >= 1') + } + + const fromMs = new Date(timestamp.from).getTime() + const toMs = new Date(timestamp.to).getTime() + + if (Number.isNaN(fromMs) || Number.isNaN(toMs)) { + throw new Error('generateActivityRelations timestamp.from/to must be valid dates') + } + + if (fromMs > toMs) { + throw new Error('generateActivityRelations timestamp.from must be <= timestamp.to') + } + + // Same channel for all rows of a given platform. + const channelByPlatform = new Map() + + return Array.from({ length: count }, (_, index) => { + const identity = identities[index % identities.length] + const platform = identity.platform as PlatformType + + // Evenly spaced across [from, to]. + const activityTimestamp = + count === 1 ? new Date(fromMs) : new Date(fromMs + ((toMs - fromMs) * index) / (count - 1)) + + let channel = channelByPlatform.get(platform) + if (!channel) { + channel = channelFor(platform) + channelByPlatform.set(platform, channel) + } + + return { + activityId: generateUUIDv1(), + memberId, + segmentId, + platform, + username: identity.value, + sourceId: generateUUIDv1(), + type: faker.helpers.arrayElement(activityTypesFor(platform)), + timestamp: activityTimestamp.toISOString(), + channel, + organizationId: resolveOrganizationId(activityTimestamp, memberOrganizations), + score: faker.number.int({ min: 0, max: 10 }), + } + }) +} + +/** Insert activityRelation rows. */ +export async function createActivityRelations( + qx: QueryExecutor, + data: ActivityRelationDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + await createOrUpdateRelations(qx, data as Parameters[1], true) + + return data +} + +function activityTypesFor(platform: PlatformType): string[] { + // Treat github-nango as github for activity types. + const key = platform === PlatformType.GITHUB_NANGO ? PlatformType.GITHUB : platform + + const types = Object.keys(DEFAULT_ACTIVITY_TYPE_SETTINGS[key] ?? {}) + + return types.length > 0 ? types : ['activity'] +} + +function channelFor(platform: PlatformType): string { + const org = faker.internet.domainWord() + const repo = faker.internet.domainWord() + + switch (platform) { + case PlatformType.GITHUB: + case PlatformType.GITHUB_NANGO: + case PlatformType.GIT: + return `https://github.com/${org}/${repo}` + + case PlatformType.GITLAB: + return `https://gitlab.com/${org}/${repo}` + + case PlatformType.DISCORD: + case PlatformType.SLACK: + return `#${faker.word.noun()}` + + default: + return `${org}/${repo}` + } +} + +/** Pick organizationId for a timestamp from the given stints. */ +function resolveOrganizationId(timestamp: Date, stints?: MemberOrganizationDbRow[]): string | null { + if (!stints?.length) { + return null + } + + let fallback: string | null = null + + for (const stint of stints) { + // No dates → keep as fallback; dated match returns immediately. + if (!stint.dateStart && !stint.dateEnd) { + fallback ??= stint.organizationId + continue + } + + const start = stint.dateStart ? new Date(stint.dateStart) : undefined + const end = stint.dateEnd ? new Date(stint.dateEnd) : undefined + + if ((!start || timestamp >= start) && (!end || timestamp <= end)) { + return stint.organizationId + } + } + + return fallback +} diff --git a/services/libs/test-kit/src/factories/defaults.ts b/services/libs/test-kit/src/factories/defaults.ts new file mode 100644 index 0000000000..085ad36f61 --- /dev/null +++ b/services/libs/test-kit/src/factories/defaults.ts @@ -0,0 +1,30 @@ +type DefaultValue = T | (() => T) + +type Defaults = { + [K in keyof T]?: DefaultValue +} + +function resolveDefault(value: DefaultValue): T { + return typeof value === 'function' ? (value as () => T)() : value +} + +function applyDefaults(row: Partial, defaults: Defaults): T { + const result = { ...row } as T + + for (const key of Object.keys(defaults) as (keyof T)[]) { + const defaultValue = defaults[key] + if (result[key] === undefined && defaultValue !== undefined) { + result[key] = resolveDefault(defaultValue as DefaultValue) + } + } + + return result +} + +/** + * Fills missing allowlisted fields on each row. Explicit values always win. + * Vue-style: primitives as values, objects/arrays (and per-row values) as factories. + */ +export function withDefaults(defaults: Defaults) { + return (rows: Partial[]): T[] => rows.map((row) => applyDefaults(row, defaults)) +} diff --git a/services/libs/test-kit/src/factories/index.ts b/services/libs/test-kit/src/factories/index.ts new file mode 100644 index 0000000000..0038d84cd0 --- /dev/null +++ b/services/libs/test-kit/src/factories/index.ts @@ -0,0 +1,10 @@ +export * from './defaults' +export * from './member' +export * from './member-identity' +export * from './organization' +export * from './organization-identity' +export * from './member-organization' +export * from './member-organization-affiliation-override' +export * from './segment' +export * from './member-segment-affiliation' +export * from './activity-relation' diff --git a/services/libs/test-kit/src/factories/member-identity.ts b/services/libs/test-kit/src/factories/member-identity.ts new file mode 100644 index 0000000000..795ec7f1bb --- /dev/null +++ b/services/libs/test-kit/src/factories/member-identity.ts @@ -0,0 +1,25 @@ +import { generateUUIDv1 } from '@crowd/common' +import { insertMemberIdentities } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import type { MemberIdentityDbInsert, MemberIdentityDbRow } from '@crowd/types' + +import { withDefaults } from './defaults' + +export const withMemberIdentityDefaults = ( + data: Partial[], +): MemberIdentityDbInsert[] => + withDefaults({ + id: () => generateUUIDv1(), + verified: true, + })(data) + +export async function createMemberIdentities( + qx: QueryExecutor, + data: MemberIdentityDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + return insertMemberIdentities(qx, data, true, true) +} diff --git a/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts b/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts new file mode 100644 index 0000000000..63a8820ea6 --- /dev/null +++ b/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts @@ -0,0 +1,26 @@ +import { changeMemberOrganizationAffiliationOverrides } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import type { + MemberOrganizationAffiliationOverrideDbInsert, + MemberOrganizationAffiliationOverrideDbRow, +} from '@crowd/types' + +export async function createMemberOrganizationAffiliationOverrides( + qx: QueryExecutor, + data: MemberOrganizationAffiliationOverrideDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + return changeMemberOrganizationAffiliationOverrides( + qx, + data.map((row) => ({ + memberId: row.memberId, + memberOrganizationId: row.memberOrganizationId, + allowAffiliation: row.allowAffiliation, + isPrimaryWorkExperience: row.isPrimaryWorkExperience, + })), + true, + ) +} diff --git a/services/libs/test-kit/src/factories/member-organization.ts b/services/libs/test-kit/src/factories/member-organization.ts new file mode 100644 index 0000000000..c7b8a59ad3 --- /dev/null +++ b/services/libs/test-kit/src/factories/member-organization.ts @@ -0,0 +1,25 @@ +import { generateUUIDv1 } from '@crowd/common' +import { insertMemberOrganizations } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import type { MemberOrganizationDbInsert, MemberOrganizationDbRow } from '@crowd/types' + +import { withDefaults } from './defaults' + +export const withMemberOrganizationDefaults = ( + data: Partial[], +): MemberOrganizationDbInsert[] => + withDefaults({ + id: () => generateUUIDv1(), + verified: false, + })(data) + +export async function createMemberOrganizations( + qx: QueryExecutor, + data: MemberOrganizationDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + return insertMemberOrganizations(qx, data, true, true) +} diff --git a/services/libs/test-kit/src/factories/member-segment-affiliation.ts b/services/libs/test-kit/src/factories/member-segment-affiliation.ts new file mode 100644 index 0000000000..216d682a7d --- /dev/null +++ b/services/libs/test-kit/src/factories/member-segment-affiliation.ts @@ -0,0 +1,25 @@ +import { generateUUIDv1 } from '@crowd/common' +import { insertMemberSegmentAffiliationRows } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import type { MemberSegmentAffiliationDbInsert, MemberSegmentAffiliationDbRow } from '@crowd/types' + +import { withDefaults } from './defaults' + +export const withMemberSegmentAffiliationDefaults = ( + data: Partial[], +): MemberSegmentAffiliationDbInsert[] => + withDefaults({ + id: () => generateUUIDv1(), + verified: false, + })(data) + +export async function createMemberSegmentAffiliations( + qx: QueryExecutor, + data: MemberSegmentAffiliationDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + return insertMemberSegmentAffiliationRows(qx, data, true, true) +} diff --git a/services/libs/test-kit/src/factories/member.ts b/services/libs/test-kit/src/factories/member.ts index e69de29bb2..e0543f58d7 100644 --- a/services/libs/test-kit/src/factories/member.ts +++ b/services/libs/test-kit/src/factories/member.ts @@ -0,0 +1,27 @@ +import { faker } from '@faker-js/faker' + +import { generateUUIDv1 } from '@crowd/common' +import { createMember as insertMember } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import type { MemberDbInsert, MemberDbRow } from '@crowd/types' + +import { withDefaults } from './defaults' + +export const withMemberDefaults = (data: Partial[]): MemberDbInsert[] => + withDefaults({ + id: () => generateUUIDv1(), + displayName: () => faker.person.fullName(), + joinedAt: () => new Date().toISOString(), + manuallyCreated: false, + })(data) + +export async function createMembers( + qx: QueryExecutor, + data: MemberDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + return Promise.all(data.map((item) => insertMember(qx, item))) +} diff --git a/services/libs/test-kit/src/factories/organization-identity.ts b/services/libs/test-kit/src/factories/organization-identity.ts new file mode 100644 index 0000000000..e047358244 --- /dev/null +++ b/services/libs/test-kit/src/factories/organization-identity.ts @@ -0,0 +1,24 @@ +import { insertOrganizationIdentities } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import type { OrganizationIdentityDbInsert, OrganizationIdentityDbRow } from '@crowd/types' + +import { withDefaults } from './defaults' + +export const withOrganizationIdentityDefaults = ( + data: Partial[], +): OrganizationIdentityDbInsert[] => + withDefaults({ + platform: 'integration', + })(data) + +/** Persist organization identity rows via the production insert path. */ +export async function createOrganizationIdentities( + qx: QueryExecutor, + data: OrganizationIdentityDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + return insertOrganizationIdentities(qx, data, true, true) +} diff --git a/services/libs/test-kit/src/factories/organization.ts b/services/libs/test-kit/src/factories/organization.ts new file mode 100644 index 0000000000..5735e9d2da --- /dev/null +++ b/services/libs/test-kit/src/factories/organization.ts @@ -0,0 +1,30 @@ +import { faker } from '@faker-js/faker' + +import { generateUUIDv1 } from '@crowd/common' +import { insertOrganizations } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import type { OrganizationDbInsert, OrganizationDbRow } from '@crowd/types' + +import { withDefaults } from './defaults' + +export const withOrganizationDefaults = ( + data: Partial[], +): OrganizationDbInsert[] => + withDefaults({ + id: () => generateUUIDv1(), + displayName: () => faker.company.name(), + manuallyCreated: false, + isTeamOrganization: false, + isAffiliationBlocked: false, + })(data) + +export async function createOrganizations( + qx: QueryExecutor, + data: OrganizationDbInsert[], +): Promise { + if (data.length === 0) { + return [] + } + + return insertOrganizations(qx, data, true, true) +} diff --git a/services/libs/test-kit/src/factories/segment.ts b/services/libs/test-kit/src/factories/segment.ts new file mode 100644 index 0000000000..6dd72d492c --- /dev/null +++ b/services/libs/test-kit/src/factories/segment.ts @@ -0,0 +1,185 @@ +import { generateUUIDv1 } from '@crowd/common' +import { insertSegments } from '@crowd/data-access-layer' +import type { QueryExecutor } from '@crowd/database' +import { type SegmentDbInsert, type SegmentDbRow, SegmentStatus } from '@crowd/types' + +import { withDefaults } from './defaults' + +/** Fields the caller may set on a node. Hierarchy fields are derived by createSegments. */ +type SegmentNodeInput = Partial< + Omit< + SegmentDbInsert, + | 'parentId' + | 'grandparentId' + | 'parentSlug' + | 'grandparentSlug' + | 'parentName' + | 'grandparentName' + > +> + +export type CreateSubproject = SegmentNodeInput +export type CreateProject = SegmentNodeInput & { + subprojects?: CreateSubproject[] +} +export type CreateProjectGroup = SegmentNodeInput & { + projects?: CreateProject[] +} + +export type CreateSegmentsResult = { + projectGroups: Array<{ + row: SegmentDbRow + projects: Array<{ + row: SegmentDbRow + subprojects: SegmentDbRow[] + }> + }> +} + +const applySegmentNodeDefaults = withDefaults({ + id: () => generateUUIDv1(), + status: SegmentStatus.ACTIVE, + isLF: true, +}) + +/** + * Fills missing allowlisted fields on each node. Does not invent children. + * Caller still owns name, slug, and tree shape. Explicit values always win. + */ +export function withSegmentDefaults(trees: CreateProjectGroup[]): CreateProjectGroup[] { + return trees.map((group) => { + const { projects, ...groupFields } = group + const [projectGroup] = applySegmentNodeDefaults([groupFields]) + + return { + ...projectGroup, + projects: projects?.map((project) => { + const { subprojects, ...projectFields } = project + const [projectDefaults] = applySegmentNodeDefaults([projectFields]) + + return { + ...projectDefaults, + subprojects: subprojects ? applySegmentNodeDefaults(subprojects) : subprojects, + } + }), + } + }) +} + +/** + * Inserts nested segment trees in one go. Derives parent/grandparent fields from + * structure (`type` is generated by the DB from parentSlug/grandparentSlug). + * Requires name + slug on every node. + */ +export async function createSegments( + qx: QueryExecutor, + trees: CreateProjectGroup[], +): Promise { + const rowsToInsert: SegmentDbInsert[] = [] + const structure: Array<{ + projectGroupId: string + projects: Array<{ projectId: string; subprojectIds: string[] }> + }> = [] + + for (const group of trees) { + const { projects: groupProjects = [], ...groupFields } = group + const projectGroupId = groupFields.id ?? generateUUIDv1() + const projectGroupName = groupFields.name + const projectGroupSlug = groupFields.slug + + if (!projectGroupName || !projectGroupSlug) { + throw new Error('createSegments requires name and slug on each project group') + } + + rowsToInsert.push({ + ...groupFields, + id: projectGroupId, + name: projectGroupName, + slug: projectGroupSlug, + parentId: null, + parentSlug: null, + parentName: null, + grandparentId: null, + grandparentSlug: null, + grandparentName: null, + }) + + const projectStructs: Array<{ projectId: string; subprojectIds: string[] }> = [] + + for (const project of groupProjects) { + const { subprojects: projectSubprojects = [], ...projectFields } = project + const projectId = projectFields.id ?? generateUUIDv1() + const projectName = projectFields.name + const projectSlug = projectFields.slug + + if (!projectName || !projectSlug) { + throw new Error('createSegments requires name and slug on each project') + } + + rowsToInsert.push({ + ...projectFields, + id: projectId, + name: projectName, + slug: projectSlug, + parentId: projectGroupId, + parentSlug: projectGroupSlug, + parentName: projectGroupName, + grandparentId: null, + grandparentSlug: null, + grandparentName: null, + }) + + const subprojectIds: string[] = [] + + for (const subproject of projectSubprojects) { + const subprojectId = subproject.id ?? generateUUIDv1() + const subprojectName = subproject.name + const subprojectSlug = subproject.slug + + if (!subprojectName || !subprojectSlug) { + throw new Error('createSegments requires name and slug on each subproject') + } + + rowsToInsert.push({ + ...subproject, + id: subprojectId, + name: subprojectName, + slug: subprojectSlug, + parentId: projectId, + parentSlug: projectSlug, + parentName: projectName, + grandparentId: projectGroupId, + grandparentSlug: projectGroupSlug, + grandparentName: projectGroupName, + }) + + subprojectIds.push(subprojectId) + } + + projectStructs.push({ projectId, subprojectIds }) + } + + structure.push({ projectGroupId, projects: projectStructs }) + } + + const rows = await insertSegments(qx, rowsToInsert, true, true) + const byId = new Map(rows.map((row) => [row.id, row])) + + const requireRow = (id: string): SegmentDbRow => { + const row = byId.get(id) + if (!row) { + throw new Error(`createSegments: inserted segment ${id} was not returned`) + } + return row + } + + return { + projectGroups: structure.map((group) => ({ + row: requireRow(group.projectGroupId), + projects: group.projects.map((project) => ({ + row: requireRow(project.projectId), + subprojects: project.subprojectIds.map((id) => requireRow(id)), + })), + })), + } +} diff --git a/services/libs/test-kit/src/faker.ts b/services/libs/test-kit/src/faker.ts new file mode 100644 index 0000000000..0601a53453 --- /dev/null +++ b/services/libs/test-kit/src/faker.ts @@ -0,0 +1 @@ +export { faker } from '@faker-js/faker' diff --git a/services/libs/test-kit/src/types.ts b/services/libs/test-kit/src/types.ts deleted file mode 100644 index 902271f076..0000000000 --- a/services/libs/test-kit/src/types.ts +++ /dev/null @@ -1,6 +0,0 @@ -export type TestPostgres = { - host: string - port: number - user: string - password: string -} diff --git a/services/libs/types/src/db/activity-relations.ts b/services/libs/types/src/db/activity-relations.ts new file mode 100644 index 0000000000..68b199a825 --- /dev/null +++ b/services/libs/types/src/db/activity-relations.ts @@ -0,0 +1,55 @@ +export interface ActivityRelationDbRow { + activityId: string + memberId: string + objectMemberId: string | null + organizationId: string | null + conversationId: string | null + parentId: string | null + segmentId: string + platform: string + username: string + objectMemberUsername: string | null + createdAt: string + updatedAt: string + sourceId: string | null + sourceParentId: string | null + type: string | null + timestamp: string | null + channel: string | null + sentimentScore: number | null + gitInsertions: number | null + gitDeletions: number | null + score: number | null + isContribution: boolean | null + pullRequestReviewState: string | null +} + +export type ActivityRelationDbInsert = Pick< + ActivityRelationDbRow, + | 'activityId' + | 'memberId' + | 'segmentId' + | 'platform' + | 'username' + | 'sourceId' + | 'type' + | 'timestamp' + | 'channel' +> & + Partial< + Omit< + ActivityRelationDbRow, + | 'activityId' + | 'memberId' + | 'segmentId' + | 'platform' + | 'username' + | 'sourceId' + | 'type' + | 'timestamp' + | 'channel' + | 'createdAt' + | 'updatedAt' + | 'isContribution' + > + > diff --git a/services/libs/types/src/db/index.ts b/services/libs/types/src/db/index.ts new file mode 100644 index 0000000000..89237068ad --- /dev/null +++ b/services/libs/types/src/db/index.ts @@ -0,0 +1,9 @@ +export * from './activity-relations' +export * from './members' +export * from './member-identities' +export * from './member-organizations' +export * from './member-organization-affiliation-overrides' +export * from './member-segment-affiliations' +export * from './organizations' +export * from './organization-identities' +export * from './segments' diff --git a/services/libs/types/src/db/member-identities.ts b/services/libs/types/src/db/member-identities.ts new file mode 100644 index 0000000000..a744249372 --- /dev/null +++ b/services/libs/types/src/db/member-identities.ts @@ -0,0 +1,36 @@ +import type { MemberIdentityType } from '../enums/members' + +export interface MemberIdentityDbRow { + id: string + memberId: string + tenantId: string + platform: string + value: string + type: MemberIdentityType + verified: boolean + verifiedBy: string | null + source: string | null + sourceId: string | null + integrationId: string | null + createdAt: string + updatedAt: string + deletedAt: string | null +} + +export type MemberIdentityDbInsert = Pick< + MemberIdentityDbRow, + 'memberId' | 'platform' | 'value' | 'type' +> & + Partial< + Omit< + MemberIdentityDbRow, + | 'memberId' + | 'platform' + | 'value' + | 'type' + | 'tenantId' + | 'createdAt' + | 'updatedAt' + | 'deletedAt' + > + > diff --git a/services/libs/types/src/db/member-organization-affiliation-overrides.ts b/services/libs/types/src/db/member-organization-affiliation-overrides.ts new file mode 100644 index 0000000000..f5082f44f1 --- /dev/null +++ b/services/libs/types/src/db/member-organization-affiliation-overrides.ts @@ -0,0 +1,13 @@ +export interface MemberOrganizationAffiliationOverrideDbRow { + id: string + memberId: string + memberOrganizationId: string + allowAffiliation: boolean | null + isPrimaryWorkExperience: boolean | null +} + +export type MemberOrganizationAffiliationOverrideDbInsert = Pick< + MemberOrganizationAffiliationOverrideDbRow, + 'memberId' | 'memberOrganizationId' +> & + Partial> diff --git a/services/libs/types/src/db/member-organizations.ts b/services/libs/types/src/db/member-organizations.ts new file mode 100644 index 0000000000..494f6f269d --- /dev/null +++ b/services/libs/types/src/db/member-organizations.ts @@ -0,0 +1,25 @@ +export interface MemberOrganizationDbRow { + id: string + memberId: string + organizationId: string + dateStart: string | null + dateEnd: string | null + title: string | null + source: string | null + verified: boolean + verifiedBy: string | null + createdAt: string + updatedAt: string + deletedAt: string | null +} + +export type MemberOrganizationDbInsert = Pick< + MemberOrganizationDbRow, + 'memberId' | 'organizationId' +> & + Partial< + Omit< + MemberOrganizationDbRow, + 'memberId' | 'organizationId' | 'createdAt' | 'updatedAt' | 'deletedAt' + > + > diff --git a/services/libs/types/src/db/member-segment-affiliations.ts b/services/libs/types/src/db/member-segment-affiliations.ts new file mode 100644 index 0000000000..b1a3c1eb08 --- /dev/null +++ b/services/libs/types/src/db/member-segment-affiliations.ts @@ -0,0 +1,17 @@ +export interface MemberSegmentAffiliationDbRow { + id: string + memberId: string + segmentId: string + organizationId: string | null + dateStart: string | null + dateEnd: string | null + verified: boolean + verifiedBy: string | null + deletedAt: string | null +} + +export type MemberSegmentAffiliationDbInsert = Pick< + MemberSegmentAffiliationDbRow, + 'memberId' | 'segmentId' +> & + Partial> diff --git a/services/libs/types/src/db/members.ts b/services/libs/types/src/db/members.ts new file mode 100644 index 0000000000..6a7cd701b0 --- /dev/null +++ b/services/libs/types/src/db/members.ts @@ -0,0 +1,29 @@ +import type { IMemberContribution } from '../members' + +export interface MemberDbRow { + id: string + displayName: string + joinedAt: string + tenantId: string + createdAt: string + updatedAt: string + manuallyCreated: boolean + attributes: Record | null + reach: Record | null + score: number | null + contributions: IMemberContribution[] | null + enrichedBy: string[] | null + manuallyChangedFields: string[] | null + importHash: string | null + deletedAt: string | null + createdById: string | null + updatedById: string | null +} + +export type MemberDbInsert = Pick & + Partial< + Omit< + MemberDbRow, + 'displayName' | 'joinedAt' | 'tenantId' | 'createdAt' | 'updatedAt' | 'deletedAt' + > + > diff --git a/services/libs/types/src/db/organization-identities.ts b/services/libs/types/src/db/organization-identities.ts new file mode 100644 index 0000000000..0d41e9bcf7 --- /dev/null +++ b/services/libs/types/src/db/organization-identities.ts @@ -0,0 +1,21 @@ +import type { OrganizationIdentityType } from '../enums/organizations' + +export interface OrganizationIdentityDbRow { + organizationId: string + platform: string + value: string + type: OrganizationIdentityType + verified: boolean + source: string | null + sourceId: string | null + integrationId: string | null + tenantId: string + createdAt: string + updatedAt: string +} + +export type OrganizationIdentityDbInsert = Pick< + OrganizationIdentityDbRow, + 'organizationId' | 'platform' | 'value' | 'type' | 'verified' +> & + Partial> diff --git a/services/libs/types/src/db/organizations.ts b/services/libs/types/src/db/organizations.ts new file mode 100644 index 0000000000..4382fe9c2c --- /dev/null +++ b/services/libs/types/src/db/organizations.ts @@ -0,0 +1,32 @@ +export interface OrganizationDbRow { + id: string + displayName: string | null + description: string | null + logo: string | null + tags: string[] | null + employees: number | null + revenueRange: Record | null + importHash: string | null + location: string | null + isTeamOrganization: boolean + isAffiliationBlocked: boolean + type: string | null + size: string | null + headline: string | null + industry: string | null + founded: number | null + employeeChurnRate: Record | null + employeeGrowthRate: Record | null + manuallyCreated: boolean + lastEnrichedAt: string | null + tenantId: string + createdAt: string + updatedAt: string + deletedAt: string | null + createdById: string | null + updatedById: string | null +} + +export type OrganizationDbInsert = Partial< + Omit +> diff --git a/services/libs/types/src/db/segments.ts b/services/libs/types/src/db/segments.ts new file mode 100644 index 0000000000..e654072752 --- /dev/null +++ b/services/libs/types/src/db/segments.ts @@ -0,0 +1,35 @@ +import type { SegmentStatus, SegmentType } from '../segments' + +export interface SegmentDbRow { + id: string + slug: string + tenantId: string + status: SegmentStatus + isLF: boolean + createdAt: string + updatedAt: string + url: string | null + name: string | null + parentName: string | null + grandparentName: string | null + parentSlug: string | null + grandparentSlug: string | null + description: string | null + sourceId: string | null + sourceParentId: string | null + customActivityTypes: Record | null + activityChannels: Record | null + dashboardCacheLastRefreshedAt: string | null + parentId: string | null + grandparentId: string | null + type: SegmentType | string | null + maturity: string | null +} + +export type SegmentDbInsert = Pick & + Partial< + Omit< + SegmentDbRow, + 'slug' | 'tenantId' | 'createdAt' | 'updatedAt' | 'dashboardCacheLastRefreshedAt' | 'type' // GENERATED ALWAYS from parentSlug/grandparentSlug + > + > diff --git a/services/libs/types/src/index.ts b/services/libs/types/src/index.ts index 6563b0e790..1e1b6095cb 100644 --- a/services/libs/types/src/index.ts +++ b/services/libs/types/src/index.ts @@ -22,6 +22,8 @@ export * from './caching' export * from './merging' +export * from './db' + export * from './errors' export * from './pubsub' diff --git a/services/libs/types/src/organizations.ts b/services/libs/types/src/organizations.ts index 85674161f8..e8d60fad14 100644 --- a/services/libs/types/src/organizations.ts +++ b/services/libs/types/src/organizations.ts @@ -142,17 +142,6 @@ export interface IOrganizationSyncRemoteData { lastSyncedAt?: string } -export interface NewOrganizationIdentity { - organizationId: string - platform: string - value: string - type: OrganizationIdentityType - verified: boolean - source: string - sourceId?: string | null - integrationId?: string | null -} - export interface IOrganizationIdentity { organizationId?: string platform: string From a57cc89d3aad33439001881328591d797c827acb Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Sun, 12 Jul 2026 22:04:47 +0530 Subject: [PATCH 2/9] fix: make prettier and linter happy Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- services/libs/data-access-layer/src/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/libs/data-access-layer/src/index.ts b/services/libs/data-access-layer/src/index.ts index 3037d24c91..9079622755 100644 --- a/services/libs/data-access-layer/src/index.ts +++ b/services/libs/data-access-layer/src/index.ts @@ -23,6 +23,9 @@ export * from './osspckgs/maintainers' export * from './osspckgs/packages' export * from './osspckgs/repos' export * from './osspckgs/stewardships' +export * from './osspckgs/types' export * from './osspckgs/versions' export * from './osspckgs/nuget' +export * from './osspckgs/downloads' +export * from './osspckgs/rubygems' export * from './osspckgs/api' From b2aca6e3d205352a4605675ec9befe8d85f768bd Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Sun, 12 Jul 2026 22:49:00 +0530 Subject: [PATCH 3/9] fix: resolve pr review comments Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- ...07-test-factory-primitives-and-defaults.md | 2 +- .../member-organization-affiliation/index.ts | 76 ++++++++++++++++--- .../data-access-layer/src/members/base.ts | 9 ++- .../data-access-layer/src/segments/index.ts | 4 + .../src/factories/activity-relation.ts | 11 +-- .../libs/test-kit/src/factories/defaults.ts | 40 ++++++---- .../test-kit/src/factories/member-identity.ts | 10 +-- ...ember-organization-affiliation-override.ts | 1 + .../src/factories/member-organization.ts | 11 --- .../factories/member-segment-affiliation.ts | 11 --- .../libs/test-kit/src/factories/member.ts | 13 ++-- .../src/factories/organization-identity.ts | 9 --- .../test-kit/src/factories/organization.ts | 17 ++--- .../libs/test-kit/src/factories/segment.ts | 23 ++---- services/libs/types/src/db/segments.ts | 2 +- services/libs/types/src/members.ts | 5 +- 16 files changed, 137 insertions(+), 107 deletions(-) diff --git a/docs/adr/0007-test-factory-primitives-and-defaults.md b/docs/adr/0007-test-factory-primitives-and-defaults.md index eaa5dc1f59..948f9d6553 100644 --- a/docs/adr/0007-test-factory-primitives-and-defaults.md +++ b/docs/adr/0007-test-factory-primitives-and-defaults.md @@ -8,7 +8,7 @@ Shared test factories are easy to turn into hidden scenario builders: they invent data the test never declared, blur what is under assertion, and drift from the real insert shapes used in production code (see ADR-0006). -We need a simple, durable rule for how test data is created — what the factory does, what defaults may fill, and what the test must own — so the pattern does not divert as more factories are added. +We need a simple, durable rule for how test data is created — what the factory does, what defaults may fill, and what the test must own — so the pattern does not diverge as more factories are added. ## Decision diff --git a/services/libs/data-access-layer/src/member-organization-affiliation/index.ts b/services/libs/data-access-layer/src/member-organization-affiliation/index.ts index 0ccfcd8198..9f8ecc179c 100644 --- a/services/libs/data-access-layer/src/member-organization-affiliation/index.ts +++ b/services/libs/data-access-layer/src/member-organization-affiliation/index.ts @@ -539,7 +539,17 @@ export async function changeMemberOrganizationAffiliationOverrides( return returnRows ? [] : undefined } - const rows: IMemberOrganizationAffiliationOverride[] = [] + type OverrideWriteRow = { + id: string + memberId: string + memberOrganizationId: string + allowAffiliation: boolean | null + isPrimaryWorkExperience: boolean | null + setAllowAffiliation: boolean + setIsPrimaryWorkExperience: boolean + } + + const rows: OverrideWriteRow[] = [] for (const d of data) { if ( @@ -551,11 +561,14 @@ export async function changeMemberOrganizationAffiliationOverrides( } rows.push({ - id: uuid(), + id: d.id ?? uuid(), memberId: d.memberId, memberOrganizationId: d.memberOrganizationId, - allowAffiliation: d.allowAffiliation, - isPrimaryWorkExperience: d.isPrimaryWorkExperience, + // undefined → null on insert; explicit null stays null + allowAffiliation: d.allowAffiliation ?? null, + isPrimaryWorkExperience: d.isPrimaryWorkExperience ?? null, + setAllowAffiliation: d.allowAffiliation !== undefined, + setIsPrimaryWorkExperience: d.isPrimaryWorkExperience !== undefined, }) } @@ -567,11 +580,13 @@ export async function changeMemberOrganizationAffiliationOverrides( .map( (_, i) => ` ( - $(id_${i}), - $(memberId_${i}), - $(memberOrganizationId_${i}), - $(allowAffiliation_${i}), - $(isPrimaryWorkExperience_${i}) + $(id_${i})::uuid, + $(memberId_${i})::uuid, + $(memberOrganizationId_${i})::uuid, + $(allowAffiliation_${i})::boolean, + $(isPrimaryWorkExperience_${i})::boolean, + $(setAllowAffiliation_${i})::boolean, + $(setIsPrimaryWorkExperience_${i})::boolean ) `, ) @@ -584,12 +599,27 @@ export async function changeMemberOrganizationAffiliationOverrides( acc[`memberOrganizationId_${i}`] = row.memberOrganizationId acc[`allowAffiliation_${i}`] = row.allowAffiliation acc[`isPrimaryWorkExperience_${i}`] = row.isPrimaryWorkExperience + acc[`setAllowAffiliation_${i}`] = row.setAllowAffiliation + acc[`setIsPrimaryWorkExperience_${i}`] = row.setIsPrimaryWorkExperience return acc }, {} as Record, ) + // CTE carries per-row "was this field provided?" flags so ON CONFLICT can + // distinguish omit (keep existing) from explicit null (clear). const query = ` + WITH input ( + id, + "memberId", + "memberOrganizationId", + "allowAffiliation", + "isPrimaryWorkExperience", + "setAllowAffiliation", + "setIsPrimaryWorkExperience" + ) AS ( + VALUES ${valuesSql} + ) INSERT INTO "memberOrganizationAffiliationOverrides" ( id, "memberId", @@ -597,11 +627,33 @@ export async function changeMemberOrganizationAffiliationOverrides( "allowAffiliation", "isPrimaryWorkExperience" ) - VALUES ${valuesSql} + SELECT + id, + "memberId", + "memberOrganizationId", + "allowAffiliation", + "isPrimaryWorkExperience" + FROM input ON CONFLICT ("memberId", "memberOrganizationId") DO UPDATE SET - "allowAffiliation" = COALESCE(EXCLUDED."allowAffiliation", "memberOrganizationAffiliationOverrides"."allowAffiliation"), - "isPrimaryWorkExperience" = COALESCE(EXCLUDED."isPrimaryWorkExperience", "memberOrganizationAffiliationOverrides"."isPrimaryWorkExperience") + "allowAffiliation" = CASE + WHEN ( + SELECT i."setAllowAffiliation" + FROM input i + WHERE i."memberId" = EXCLUDED."memberId" + AND i."memberOrganizationId" = EXCLUDED."memberOrganizationId" + ) THEN EXCLUDED."allowAffiliation" + ELSE "memberOrganizationAffiliationOverrides"."allowAffiliation" + END, + "isPrimaryWorkExperience" = CASE + WHEN ( + SELECT i."setIsPrimaryWorkExperience" + FROM input i + WHERE i."memberId" = EXCLUDED."memberId" + AND i."memberOrganizationId" = EXCLUDED."memberOrganizationId" + ) THEN EXCLUDED."isPrimaryWorkExperience" + ELSE "memberOrganizationAffiliationOverrides"."isPrimaryWorkExperience" + END ${returnRows ? 'RETURNING *' : ''} ` diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 9f9ede74d7..ae7d8003a9 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -757,7 +757,14 @@ export async function createMember(qx: QueryExecutor, data: MemberDbInsert): Pro const id = data.id ?? generateUUIDv1() const ts = new Date() const dbInstance = getDbInstance() - const columnSet = new dbInstance.helpers.ColumnSet(MEMBER_INSERT_COLUMNS, { + + // Omit `score` when unset so Postgres keeps DEFAULT -1 (undefined would insert NULL). + let columns = MEMBER_INSERT_COLUMNS + if (data.score === undefined) { + columns = MEMBER_INSERT_COLUMNS.filter((column) => column !== 'score') + } + + const columnSet = new dbInstance.helpers.ColumnSet(columns, { table: { table: 'members', }, diff --git a/services/libs/data-access-layer/src/segments/index.ts b/services/libs/data-access-layer/src/segments/index.ts index fe862309e9..7c8d52df4e 100644 --- a/services/libs/data-access-layer/src/segments/index.ts +++ b/services/libs/data-access-layer/src/segments/index.ts @@ -186,6 +186,10 @@ export async function insertSegments( failOnConflict = false, returnRows = false, ): Promise { + if (segments.length === 0) { + return returnRows ? [] : 0 + } + const ts = new Date() const query = prepareBulkInsert( diff --git a/services/libs/test-kit/src/factories/activity-relation.ts b/services/libs/test-kit/src/factories/activity-relation.ts index 962a5276d6..73c1edea65 100644 --- a/services/libs/test-kit/src/factories/activity-relation.ts +++ b/services/libs/test-kit/src/factories/activity-relation.ts @@ -27,13 +27,10 @@ interface GenerateActivityRelationsInput { memberOrganizations?: MemberOrganizationDbRow[] } -export const withActivityRelationDefaults = ( - data: Partial[], -): ActivityRelationDbInsert[] => - withDefaults({ - activityId: () => generateUUIDv1(), - sourceId: () => generateUUIDv1(), - })(data) +export const withActivityRelationDefaults = withDefaults()({ + activityId: () => generateUUIDv1(), + sourceId: () => generateUUIDv1(), +}) /** Build N activityRelation rows from member context. */ export function generateActivityRelations( diff --git a/services/libs/test-kit/src/factories/defaults.ts b/services/libs/test-kit/src/factories/defaults.ts index 085ad36f61..d0938b84eb 100644 --- a/services/libs/test-kit/src/factories/defaults.ts +++ b/services/libs/test-kit/src/factories/defaults.ts @@ -1,20 +1,28 @@ -type DefaultValue = T | (() => T) +type DefaultValue = V | (() => V) -type Defaults = { - [K in keyof T]?: DefaultValue +type DefaultsFor = { + [P in K]: DefaultValue } -function resolveDefault(value: DefaultValue): T { - return typeof value === 'function' ? (value as () => T)() : value +/** Required keys of T stay required; keys covered by defaults become optional. */ +type WithDefaultsRow = Omit & Partial> + +function resolveDefault(value: DefaultValue): V { + if (typeof value === 'function') { + return (value as () => V)() + } + return value as V } -function applyDefaults(row: Partial, defaults: Defaults): T { +function applyDefaults( + row: WithDefaultsRow, + defaults: DefaultsFor, +): T { const result = { ...row } as T - for (const key of Object.keys(defaults) as (keyof T)[]) { - const defaultValue = defaults[key] - if (result[key] === undefined && defaultValue !== undefined) { - result[key] = resolveDefault(defaultValue as DefaultValue) + for (const key of Object.keys(defaults) as K[]) { + if (result[key] === undefined) { + result[key] = resolveDefault(defaults[key]) as T[K] } } @@ -23,8 +31,14 @@ function applyDefaults(row: Partial, defaults: Defaults) /** * Fills missing allowlisted fields on each row. Explicit values always win. - * Vue-style: primitives as values, objects/arrays (and per-row values) as factories. + * Keys provided in `defaults` become optional on input; all other required + * keys of T remain required at compile time. + * + * Usage: `withDefaults()({ id: () => ..., displayName: () => ... })` */ -export function withDefaults(defaults: Defaults) { - return (rows: Partial[]): T[] => rows.map((row) => applyDefaults(row, defaults)) +export function withDefaults() { + return (defaults: DefaultsFor) => { + return (rows: Array>): T[] => + rows.map((row) => applyDefaults(row, defaults)) + } } diff --git a/services/libs/test-kit/src/factories/member-identity.ts b/services/libs/test-kit/src/factories/member-identity.ts index 795ec7f1bb..e5a88df7a0 100644 --- a/services/libs/test-kit/src/factories/member-identity.ts +++ b/services/libs/test-kit/src/factories/member-identity.ts @@ -5,13 +5,9 @@ import type { MemberIdentityDbInsert, MemberIdentityDbRow } from '@crowd/types' import { withDefaults } from './defaults' -export const withMemberIdentityDefaults = ( - data: Partial[], -): MemberIdentityDbInsert[] => - withDefaults({ - id: () => generateUUIDv1(), - verified: true, - })(data) +export const withMemberIdentityDefaults = withDefaults()({ + id: () => generateUUIDv1(), +}) export async function createMemberIdentities( qx: QueryExecutor, diff --git a/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts b/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts index 63a8820ea6..73daa90fd7 100644 --- a/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts +++ b/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts @@ -16,6 +16,7 @@ export async function createMemberOrganizationAffiliationOverrides( return changeMemberOrganizationAffiliationOverrides( qx, data.map((row) => ({ + id: row.id, memberId: row.memberId, memberOrganizationId: row.memberOrganizationId, allowAffiliation: row.allowAffiliation, diff --git a/services/libs/test-kit/src/factories/member-organization.ts b/services/libs/test-kit/src/factories/member-organization.ts index c7b8a59ad3..8b3f6b4c71 100644 --- a/services/libs/test-kit/src/factories/member-organization.ts +++ b/services/libs/test-kit/src/factories/member-organization.ts @@ -1,18 +1,7 @@ -import { generateUUIDv1 } from '@crowd/common' import { insertMemberOrganizations } from '@crowd/data-access-layer' import type { QueryExecutor } from '@crowd/database' import type { MemberOrganizationDbInsert, MemberOrganizationDbRow } from '@crowd/types' -import { withDefaults } from './defaults' - -export const withMemberOrganizationDefaults = ( - data: Partial[], -): MemberOrganizationDbInsert[] => - withDefaults({ - id: () => generateUUIDv1(), - verified: false, - })(data) - export async function createMemberOrganizations( qx: QueryExecutor, data: MemberOrganizationDbInsert[], diff --git a/services/libs/test-kit/src/factories/member-segment-affiliation.ts b/services/libs/test-kit/src/factories/member-segment-affiliation.ts index 216d682a7d..9966e1e816 100644 --- a/services/libs/test-kit/src/factories/member-segment-affiliation.ts +++ b/services/libs/test-kit/src/factories/member-segment-affiliation.ts @@ -1,18 +1,7 @@ -import { generateUUIDv1 } from '@crowd/common' import { insertMemberSegmentAffiliationRows } from '@crowd/data-access-layer' import type { QueryExecutor } from '@crowd/database' import type { MemberSegmentAffiliationDbInsert, MemberSegmentAffiliationDbRow } from '@crowd/types' -import { withDefaults } from './defaults' - -export const withMemberSegmentAffiliationDefaults = ( - data: Partial[], -): MemberSegmentAffiliationDbInsert[] => - withDefaults({ - id: () => generateUUIDv1(), - verified: false, - })(data) - export async function createMemberSegmentAffiliations( qx: QueryExecutor, data: MemberSegmentAffiliationDbInsert[], diff --git a/services/libs/test-kit/src/factories/member.ts b/services/libs/test-kit/src/factories/member.ts index e0543f58d7..8bdbcaf57d 100644 --- a/services/libs/test-kit/src/factories/member.ts +++ b/services/libs/test-kit/src/factories/member.ts @@ -7,13 +7,12 @@ import type { MemberDbInsert, MemberDbRow } from '@crowd/types' import { withDefaults } from './defaults' -export const withMemberDefaults = (data: Partial[]): MemberDbInsert[] => - withDefaults({ - id: () => generateUUIDv1(), - displayName: () => faker.person.fullName(), - joinedAt: () => new Date().toISOString(), - manuallyCreated: false, - })(data) +export const withMemberDefaults = withDefaults()({ + id: () => generateUUIDv1(), + displayName: () => faker.person.fullName(), + joinedAt: () => new Date().toISOString(), + manuallyCreated: false, +}) export async function createMembers( qx: QueryExecutor, diff --git a/services/libs/test-kit/src/factories/organization-identity.ts b/services/libs/test-kit/src/factories/organization-identity.ts index e047358244..9a724cf05b 100644 --- a/services/libs/test-kit/src/factories/organization-identity.ts +++ b/services/libs/test-kit/src/factories/organization-identity.ts @@ -2,15 +2,6 @@ import { insertOrganizationIdentities } from '@crowd/data-access-layer' import type { QueryExecutor } from '@crowd/database' import type { OrganizationIdentityDbInsert, OrganizationIdentityDbRow } from '@crowd/types' -import { withDefaults } from './defaults' - -export const withOrganizationIdentityDefaults = ( - data: Partial[], -): OrganizationIdentityDbInsert[] => - withDefaults({ - platform: 'integration', - })(data) - /** Persist organization identity rows via the production insert path. */ export async function createOrganizationIdentities( qx: QueryExecutor, diff --git a/services/libs/test-kit/src/factories/organization.ts b/services/libs/test-kit/src/factories/organization.ts index 5735e9d2da..7e65b16961 100644 --- a/services/libs/test-kit/src/factories/organization.ts +++ b/services/libs/test-kit/src/factories/organization.ts @@ -7,16 +7,13 @@ import type { OrganizationDbInsert, OrganizationDbRow } from '@crowd/types' import { withDefaults } from './defaults' -export const withOrganizationDefaults = ( - data: Partial[], -): OrganizationDbInsert[] => - withDefaults({ - id: () => generateUUIDv1(), - displayName: () => faker.company.name(), - manuallyCreated: false, - isTeamOrganization: false, - isAffiliationBlocked: false, - })(data) +export const withOrganizationDefaults = withDefaults()({ + id: () => generateUUIDv1(), + displayName: () => faker.company.name(), + manuallyCreated: false, + isTeamOrganization: false, + isAffiliationBlocked: false, +}) export async function createOrganizations( qx: QueryExecutor, diff --git a/services/libs/test-kit/src/factories/segment.ts b/services/libs/test-kit/src/factories/segment.ts index 6dd72d492c..c4b2eed26f 100644 --- a/services/libs/test-kit/src/factories/segment.ts +++ b/services/libs/test-kit/src/factories/segment.ts @@ -5,10 +5,15 @@ import { type SegmentDbInsert, type SegmentDbRow, SegmentStatus } from '@crowd/t import { withDefaults } from './defaults' -/** Fields the caller may set on a node. Hierarchy fields are derived by createSegments. */ -type SegmentNodeInput = Partial< +/** Hierarchy fields are derived by createSegments; callers must supply name + slug. */ +type SegmentNodeInput = { + name: string + slug: string +} & Partial< Omit< SegmentDbInsert, + | 'name' + | 'slug' | 'parentId' | 'grandparentId' | 'parentSlug' @@ -36,7 +41,7 @@ export type CreateSegmentsResult = { }> } -const applySegmentNodeDefaults = withDefaults({ +const applySegmentNodeDefaults = withDefaults()({ id: () => generateUUIDv1(), status: SegmentStatus.ACTIVE, isLF: true, @@ -87,10 +92,6 @@ export async function createSegments( const projectGroupName = groupFields.name const projectGroupSlug = groupFields.slug - if (!projectGroupName || !projectGroupSlug) { - throw new Error('createSegments requires name and slug on each project group') - } - rowsToInsert.push({ ...groupFields, id: projectGroupId, @@ -112,10 +113,6 @@ export async function createSegments( const projectName = projectFields.name const projectSlug = projectFields.slug - if (!projectName || !projectSlug) { - throw new Error('createSegments requires name and slug on each project') - } - rowsToInsert.push({ ...projectFields, id: projectId, @@ -136,10 +133,6 @@ export async function createSegments( const subprojectName = subproject.name const subprojectSlug = subproject.slug - if (!subprojectName || !subprojectSlug) { - throw new Error('createSegments requires name and slug on each subproject') - } - rowsToInsert.push({ ...subproject, id: subprojectId, diff --git a/services/libs/types/src/db/segments.ts b/services/libs/types/src/db/segments.ts index e654072752..788a126185 100644 --- a/services/libs/types/src/db/segments.ts +++ b/services/libs/types/src/db/segments.ts @@ -22,7 +22,7 @@ export interface SegmentDbRow { dashboardCacheLastRefreshedAt: string | null parentId: string | null grandparentId: string | null - type: SegmentType | string | null + type: SegmentType maturity: string | null } diff --git a/services/libs/types/src/members.ts b/services/libs/types/src/members.ts index fc31cacdf4..fbd231f169 100644 --- a/services/libs/types/src/members.ts +++ b/services/libs/types/src/members.ts @@ -227,8 +227,9 @@ export interface IMemberOpensearch { } export interface IChangeAffiliationOverrideData { - allowAffiliation?: boolean - isPrimaryWorkExperience?: boolean + id?: string + allowAffiliation?: boolean | null + isPrimaryWorkExperience?: boolean | null memberOrganizationId: string memberId: string } From 79a8ad3a3df2d28dac8ba08f88130b84ead797a5 Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:48:01 +0530 Subject: [PATCH 4/9] refactor: nitpicks after self pr review Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- .../patchProjectAffiliation.ts | 4 +- .../member-organization-affiliation/index.ts | 74 +++---------------- .../src/member_segment_affiliations/index.ts | 18 ++--- .../data-access-layer/src/members/base.ts | 22 +++--- .../src/members/projectAffiliations.ts | 5 +- ...ember-organization-affiliation-override.ts | 6 +- .../factories/member-segment-affiliation.ts | 4 +- services/libs/types/src/members.ts | 4 +- 8 files changed, 39 insertions(+), 98 deletions(-) diff --git a/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts b/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts index 1623d4ea0c..b432b07058 100644 --- a/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts +++ b/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts @@ -10,7 +10,7 @@ import { fetchMemberSegmentAffiliationsForProject, findMaintainerRoles, findMemberById, - insertMemberSegmentAffiliations, + insertVerifiedMemberSegmentAffiliations, } from '@crowd/data-access-layer' import type { ISegmentAffiliationWithOrg } from '@crowd/data-access-layer' import { deleteMemberSegmentAffiliations } from '@crowd/data-access-layer/src/member_segment_affiliations' @@ -83,7 +83,7 @@ export async function patchProjectAffiliation(req: Request, res: Response): Prom await deleteMemberSegmentAffiliations(tx, { memberId, segmentId: projectId }) if (affiliations.length > 0) { - await insertMemberSegmentAffiliations( + await insertVerifiedMemberSegmentAffiliations( tx, memberId, projectId, diff --git a/services/libs/data-access-layer/src/member-organization-affiliation/index.ts b/services/libs/data-access-layer/src/member-organization-affiliation/index.ts index 9f8ecc179c..a81c54c3ae 100644 --- a/services/libs/data-access-layer/src/member-organization-affiliation/index.ts +++ b/services/libs/data-access-layer/src/member-organization-affiliation/index.ts @@ -539,17 +539,7 @@ export async function changeMemberOrganizationAffiliationOverrides( return returnRows ? [] : undefined } - type OverrideWriteRow = { - id: string - memberId: string - memberOrganizationId: string - allowAffiliation: boolean | null - isPrimaryWorkExperience: boolean | null - setAllowAffiliation: boolean - setIsPrimaryWorkExperience: boolean - } - - const rows: OverrideWriteRow[] = [] + const rows: IMemberOrganizationAffiliationOverride[] = [] for (const d of data) { if ( @@ -564,11 +554,8 @@ export async function changeMemberOrganizationAffiliationOverrides( id: d.id ?? uuid(), memberId: d.memberId, memberOrganizationId: d.memberOrganizationId, - // undefined → null on insert; explicit null stays null - allowAffiliation: d.allowAffiliation ?? null, - isPrimaryWorkExperience: d.isPrimaryWorkExperience ?? null, - setAllowAffiliation: d.allowAffiliation !== undefined, - setIsPrimaryWorkExperience: d.isPrimaryWorkExperience !== undefined, + allowAffiliation: d.allowAffiliation, + isPrimaryWorkExperience: d.isPrimaryWorkExperience, }) } @@ -580,13 +567,11 @@ export async function changeMemberOrganizationAffiliationOverrides( .map( (_, i) => ` ( - $(id_${i})::uuid, - $(memberId_${i})::uuid, - $(memberOrganizationId_${i})::uuid, - $(allowAffiliation_${i})::boolean, - $(isPrimaryWorkExperience_${i})::boolean, - $(setAllowAffiliation_${i})::boolean, - $(setIsPrimaryWorkExperience_${i})::boolean + $(id_${i}), + $(memberId_${i}), + $(memberOrganizationId_${i}), + $(allowAffiliation_${i}), + $(isPrimaryWorkExperience_${i}) ) `, ) @@ -599,27 +584,12 @@ export async function changeMemberOrganizationAffiliationOverrides( acc[`memberOrganizationId_${i}`] = row.memberOrganizationId acc[`allowAffiliation_${i}`] = row.allowAffiliation acc[`isPrimaryWorkExperience_${i}`] = row.isPrimaryWorkExperience - acc[`setAllowAffiliation_${i}`] = row.setAllowAffiliation - acc[`setIsPrimaryWorkExperience_${i}`] = row.setIsPrimaryWorkExperience return acc }, {} as Record, ) - // CTE carries per-row "was this field provided?" flags so ON CONFLICT can - // distinguish omit (keep existing) from explicit null (clear). const query = ` - WITH input ( - id, - "memberId", - "memberOrganizationId", - "allowAffiliation", - "isPrimaryWorkExperience", - "setAllowAffiliation", - "setIsPrimaryWorkExperience" - ) AS ( - VALUES ${valuesSql} - ) INSERT INTO "memberOrganizationAffiliationOverrides" ( id, "memberId", @@ -627,33 +597,11 @@ export async function changeMemberOrganizationAffiliationOverrides( "allowAffiliation", "isPrimaryWorkExperience" ) - SELECT - id, - "memberId", - "memberOrganizationId", - "allowAffiliation", - "isPrimaryWorkExperience" - FROM input + VALUES ${valuesSql} ON CONFLICT ("memberId", "memberOrganizationId") DO UPDATE SET - "allowAffiliation" = CASE - WHEN ( - SELECT i."setAllowAffiliation" - FROM input i - WHERE i."memberId" = EXCLUDED."memberId" - AND i."memberOrganizationId" = EXCLUDED."memberOrganizationId" - ) THEN EXCLUDED."allowAffiliation" - ELSE "memberOrganizationAffiliationOverrides"."allowAffiliation" - END, - "isPrimaryWorkExperience" = CASE - WHEN ( - SELECT i."setIsPrimaryWorkExperience" - FROM input i - WHERE i."memberId" = EXCLUDED."memberId" - AND i."memberOrganizationId" = EXCLUDED."memberOrganizationId" - ) THEN EXCLUDED."isPrimaryWorkExperience" - ELSE "memberOrganizationAffiliationOverrides"."isPrimaryWorkExperience" - END + "allowAffiliation" = COALESCE(EXCLUDED."allowAffiliation", "memberOrganizationAffiliationOverrides"."allowAffiliation"), + "isPrimaryWorkExperience" = COALESCE(EXCLUDED."isPrimaryWorkExperience", "memberOrganizationAffiliationOverrides"."isPrimaryWorkExperience") ${returnRows ? 'RETURNING *' : ''} ` diff --git a/services/libs/data-access-layer/src/member_segment_affiliations/index.ts b/services/libs/data-access-layer/src/member_segment_affiliations/index.ts index 97af82a1cf..b8f0ecaf99 100644 --- a/services/libs/data-access-layer/src/member_segment_affiliations/index.ts +++ b/services/libs/data-access-layer/src/member_segment_affiliations/index.ts @@ -81,19 +81,19 @@ export async function findMemberAffiliations( ) } -export async function insertMemberSegmentAffiliationRows( +export async function insertMemberSegmentAffiliations( qx: QueryExecutor, affiliations: MemberSegmentAffiliationDbInsert[], failOnConflict: boolean, returnRows: true, ): Promise -export async function insertMemberSegmentAffiliationRows( +export async function insertMemberSegmentAffiliations( qx: QueryExecutor, affiliations: MemberSegmentAffiliationDbInsert[], failOnConflict?: boolean, returnRows?: false, ): Promise -export async function insertMemberSegmentAffiliationRows( +export async function insertMemberSegmentAffiliations( qx: QueryExecutor, affiliations: MemberSegmentAffiliationDbInsert[], failOnConflict = false, @@ -116,14 +116,10 @@ export async function insertMemberSegmentAffiliationRows( 'verifiedBy', ], affiliations.map((a) => ({ + ...a, id: a.id ?? generateUUIDv1(), - memberId: a.memberId, - segmentId: a.segmentId, - organizationId: a.organizationId ?? null, - dateStart: a.dateStart ?? null, - dateEnd: a.dateEnd ?? null, + // NOT NULL column — must set explicitly while listed in INSERT, else undefined → NULL verified: a.verified ?? false, - verifiedBy: a.verifiedBy ?? null, })), failOnConflict ? undefined : 'DO NOTHING', returnRows, @@ -136,10 +132,10 @@ export async function insertMemberSegmentAffiliationRows( return qx.result(query) } -/** @deprecated Prefer `insertMemberSegmentAffiliationRows` with full insert payloads. */ +/** @deprecated Prefer `insertMemberSegmentAffiliations` with full insert payloads. */ // eslint-disable-next-line @typescript-eslint/no-explicit-any export async function insertMemberAffiliations(qx: QueryExecutor, memberId: string, data: any[]) { - return insertMemberSegmentAffiliationRows( + return insertMemberSegmentAffiliations( qx, data.map((item) => ({ memberId, diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index ae7d8003a9..eb5fbd37c3 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -758,18 +758,6 @@ export async function createMember(qx: QueryExecutor, data: MemberDbInsert): Pro const ts = new Date() const dbInstance = getDbInstance() - // Omit `score` when unset so Postgres keeps DEFAULT -1 (undefined would insert NULL). - let columns = MEMBER_INSERT_COLUMNS - if (data.score === undefined) { - columns = MEMBER_INSERT_COLUMNS.filter((column) => column !== 'score') - } - - const columnSet = new dbInstance.helpers.ColumnSet(columns, { - table: { - table: 'members', - }, - }) - const dbData: Record = { ...data, id, @@ -779,6 +767,16 @@ export async function createMember(qx: QueryExecutor, data: MemberDbInsert): Pro updatedAt: ts, } + // Omit unset columns so Postgres DEFAULT applies (listing a column with + // undefined/null bypasses DEFAULT and inserts NULL instead). + const columns = MEMBER_INSERT_COLUMNS.filter((column) => dbData[column] !== undefined) + + const columnSet = new dbInstance.helpers.ColumnSet(columns, { + table: { + table: 'members', + }, + }) + if (Array.isArray(dbData.contributions)) { dbData.contributions = JSON.stringify(dbData.contributions) } diff --git a/services/libs/data-access-layer/src/members/projectAffiliations.ts b/services/libs/data-access-layer/src/members/projectAffiliations.ts index 92a3ebb08a..5e472d1299 100644 --- a/services/libs/data-access-layer/src/members/projectAffiliations.ts +++ b/services/libs/data-access-layer/src/members/projectAffiliations.ts @@ -129,10 +129,9 @@ export interface ISegmentAffiliationInsert { } /** - * Insert multiple segment affiliations for a member + project (segment) combination. - * All inserted affiliations are marked as verified. + * Insert multiple verified segment affiliations for a member + project (segment) combination. */ -export async function insertMemberSegmentAffiliations( +export async function insertVerifiedMemberSegmentAffiliations( qx: QueryExecutor, memberId: string, segmentId: string, diff --git a/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts b/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts index 73daa90fd7..68976c168b 100644 --- a/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts +++ b/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts @@ -5,7 +5,7 @@ import type { MemberOrganizationAffiliationOverrideDbRow, } from '@crowd/types' -export async function createMemberOrganizationAffiliationOverrides( +export async function upsertMemberOrganizationAffiliationOverrides( qx: QueryExecutor, data: MemberOrganizationAffiliationOverrideDbInsert[], ): Promise { @@ -19,8 +19,8 @@ export async function createMemberOrganizationAffiliationOverrides( id: row.id, memberId: row.memberId, memberOrganizationId: row.memberOrganizationId, - allowAffiliation: row.allowAffiliation, - isPrimaryWorkExperience: row.isPrimaryWorkExperience, + allowAffiliation: row.allowAffiliation ?? undefined, + isPrimaryWorkExperience: row.isPrimaryWorkExperience ?? undefined, })), true, ) diff --git a/services/libs/test-kit/src/factories/member-segment-affiliation.ts b/services/libs/test-kit/src/factories/member-segment-affiliation.ts index 9966e1e816..01a11ee73a 100644 --- a/services/libs/test-kit/src/factories/member-segment-affiliation.ts +++ b/services/libs/test-kit/src/factories/member-segment-affiliation.ts @@ -1,4 +1,4 @@ -import { insertMemberSegmentAffiliationRows } from '@crowd/data-access-layer' +import { insertMemberSegmentAffiliations } from '@crowd/data-access-layer' import type { QueryExecutor } from '@crowd/database' import type { MemberSegmentAffiliationDbInsert, MemberSegmentAffiliationDbRow } from '@crowd/types' @@ -10,5 +10,5 @@ export async function createMemberSegmentAffiliations( return [] } - return insertMemberSegmentAffiliationRows(qx, data, true, true) + return insertMemberSegmentAffiliations(qx, data, true, true) } diff --git a/services/libs/types/src/members.ts b/services/libs/types/src/members.ts index fbd231f169..a823bc06cb 100644 --- a/services/libs/types/src/members.ts +++ b/services/libs/types/src/members.ts @@ -228,8 +228,8 @@ export interface IMemberOpensearch { export interface IChangeAffiliationOverrideData { id?: string - allowAffiliation?: boolean | null - isPrimaryWorkExperience?: boolean | null + allowAffiliation?: boolean + isPrimaryWorkExperience?: boolean memberOrganizationId: string memberId: string } From dc21b991c6bd1e75321368ca910ecb455c1d5b3e Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:57:08 +0530 Subject: [PATCH 5/9] refactor: simplify dal inserts and drop createMemberIdentity Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- .../identities/createMemberIdentity.ts | 25 +++++++------ .../database/repositories/memberRepository.ts | 34 ++++++++++++------ .../services/member/memberIdentityService.ts | 11 +++--- .../src/activities/enrichment.ts | 30 ++++++++-------- .../src/activities/member.ts | 23 ++++++------ .../src/services/member/unmerge.ts | 27 +++++++------- .../src/member_segment_affiliations/index.ts | 1 - .../src/members/identities.ts | 36 +------------------ .../src/members/organizations.ts | 12 +++---- .../src/organizations/base.ts | 25 +++---------- .../src/organizations/identities.ts | 9 +---- .../data-access-layer/src/segments/index.ts | 25 ++----------- ...ember-organization-affiliation-override.ts | 12 +------ ...mber-organization-affiliation-overrides.ts | 9 ++++- 14 files changed, 108 insertions(+), 171 deletions(-) diff --git a/backend/src/api/public/v1/members/identities/createMemberIdentity.ts b/backend/src/api/public/v1/members/identities/createMemberIdentity.ts index bef6c773c6..17d45a9269 100644 --- a/backend/src/api/public/v1/members/identities/createMemberIdentity.ts +++ b/backend/src/api/public/v1/members/identities/createMemberIdentity.ts @@ -7,7 +7,7 @@ import { MemberField, findMemberById, findMemberIdentitiesByValue, - createMemberIdentity as insertMemberIdentity, + insertMemberIdentities, touchMemberUpdatedAt, updateMemberIdentity, } from '@crowd/data-access-layer' @@ -69,20 +69,23 @@ export async function createMemberIdentity(req: Request, res: Response): Promise alreadyExisted = true result = exactMatch } else { - result = await insertMemberIdentity( + const [created] = await insertMemberIdentities( tx, - { - memberId, - platform: data.platform, - value: normalizedValue, - type: data.type, - source: data.source, - verified: data.verified, - verifiedBy: data.verifiedBy, - }, + [ + { + memberId, + platform: data.platform, + value: normalizedValue, + type: data.type, + source: data.source, + verified: data.verified, + verifiedBy: data.verifiedBy, + }, + ], true, true, ) + result = created } // A verified identity confirms the same value for this member, so keep same-value diff --git a/backend/src/database/repositories/memberRepository.ts b/backend/src/database/repositories/memberRepository.ts index 3cc917b0e0..84d3d6465f 100644 --- a/backend/src/database/repositories/memberRepository.ts +++ b/backend/src/database/repositories/memberRepository.ts @@ -18,7 +18,7 @@ import { import { BotDetectionService, CommonMemberService } from '@crowd/common_services' import { OrganizationField, - createMemberIdentity, + insertMemberIdentities, deleteMemberIdentities, deleteMemberIdentitiesByCombinations, findAlreadyExistingVerifiedIdentities, @@ -159,8 +159,9 @@ class MemberRepository { const subprojectIds = await getSegmentSubprojectIds(qx, currentSegments) if (data.identities) { - for (const i of data.identities as IMemberIdentity[]) { - await createMemberIdentity(qx, { + await insertMemberIdentities( + qx, + (data.identities as IMemberIdentity[]).map((i) => ({ memberId: record.id, platform: i.platform, type: i.type, @@ -169,15 +170,16 @@ class MemberRepository { integrationId: i.integrationId || null, verified: i.verified, source: i.source, - }) - } + })), + ) } else if (data.username) { const username: PlatformIdentities = mapUsernameToIdentities(data.username) + const identitiesToInsert = [] for (const platform of Object.keys(username) as PlatformType[]) { const identities: any[] = username[platform] for (const identity of identities) { - await createMemberIdentity(qx, { + identitiesToInsert.push({ memberId: record.id, platform, value: identity.value ? identity.value : identity.username, @@ -189,6 +191,10 @@ class MemberRepository { }) } } + + if (identitiesToInsert.length > 0) { + await insertMemberIdentities(qx, identitiesToInsert) + } } await includeMemberToSegments(qx, record.id, subprojectIds) @@ -1049,8 +1055,9 @@ class MemberRepository { } if (data.identitiesToCreate && data.identitiesToCreate.length > 0) { - for (const i of data.identitiesToCreate) { - await createMemberIdentity(qx, { + await insertMemberIdentities( + qx, + data.identitiesToCreate.map((i) => ({ memberId: record.id, platform: i.platform, value: i.value, @@ -1059,8 +1066,8 @@ class MemberRepository { integrationId: i.integrationId || null, verified: i.verified !== undefined ? i.verified : !!manualChange, source: i.source, - }) - } + })), + ) } if (data.identitiesToUpdate && data.identitiesToUpdate.length > 0) { @@ -1094,6 +1101,7 @@ class MemberRepository { const platformsToDelete: string[] = [] const valuesToDelete: string[] = [] const typesToDelete: MemberIdentityType[] = [] + const identitiesToInsert = [] for (const platform of platforms) { const identities = data.username[platform] @@ -1112,7 +1120,7 @@ class MemberRepository { (identity.username && identity.username !== '') || (identity.value && identity.value !== '') ) { - await createMemberIdentity(qx, { + identitiesToInsert.push({ memberId: record.id, platform, value: identity.value ? identity.value : identity.username, @@ -1126,6 +1134,10 @@ class MemberRepository { } } + if (identitiesToInsert.length > 0) { + await insertMemberIdentities(qx, identitiesToInsert) + } + if (platformsToDelete.length > 0) { await deleteMemberIdentitiesByCombinations(qx, { memberId: record.id, diff --git a/backend/src/services/member/memberIdentityService.ts b/backend/src/services/member/memberIdentityService.ts index 48569bf93b..7e6801ee46 100644 --- a/backend/src/services/member/memberIdentityService.ts +++ b/backend/src/services/member/memberIdentityService.ts @@ -3,7 +3,7 @@ import lodash from 'lodash' import { captureApiChange, memberEditIdentitiesAction } from '@crowd/audit-logs' import { Error404, Error409 } from '@crowd/common' -import { createMemberIdentity, findIdentitiesForMembers } from '@crowd/data-access-layer' +import { insertMemberIdentities, findIdentitiesForMembers } from '@crowd/data-access-layer' import { deleteMemberIdentity, fetchMemberIdentities, @@ -77,7 +77,7 @@ export default class MemberIdentityService extends LoggerBase { } // Create member identity - await createMemberIdentity(qx, { ...data, memberId }) + await insertMemberIdentities(qx, [{ ...data, memberId }]) await touchMemberUpdatedAt(qx, memberId) @@ -151,9 +151,10 @@ export default class MemberIdentityService extends LoggerBase { } // Create member identities - for (const identity of data) { - await createMemberIdentity(qx, { ...identity, memberId }) - } + await insertMemberIdentities( + qx, + data.map((identity) => ({ ...identity, memberId })), + ) await touchMemberUpdatedAt(qx, memberId) diff --git a/services/apps/members_enrichment_worker/src/activities/enrichment.ts b/services/apps/members_enrichment_worker/src/activities/enrichment.ts index 9c54ade13b..1ffb8ffbc3 100644 --- a/services/apps/members_enrichment_worker/src/activities/enrichment.ts +++ b/services/apps/members_enrichment_worker/src/activities/enrichment.ts @@ -13,11 +13,11 @@ import { signalMemberUpdate } from '@crowd/common_services' import { changeMemberOrganizationAffiliationOverrides, fetchManyOrganizationAffiliationPolicies, + insertMemberIdentities, updateMemberAttributes, updateMemberContributions, updateMemberReach, } from '@crowd/data-access-layer' -import { createMemberIdentity } from '@crowd/data-access-layer' import { findMemberIdentityWithTheMostActivityInPlatform as getMemberMostActiveIdentity } from '@crowd/data-access-layer/src/activityRelations' import { deleteMemberSegmentAffiliations } from '@crowd/data-access-layer/src/member_segment_affiliations' import { getPlatformPriorityArray } from '@crowd/data-access-layer/src/members/attributeSettings' @@ -308,21 +308,19 @@ export async function updateMemberUsingSquashedPayload( // process identities if (squashedPayload.identities.length > 0) { svc.log.debug({ memberId }, 'Adding to member identities!') - for (const i of squashedPayload.identities) { - didUpdate = true - await createMemberIdentity( - qx, - { - memberId, - platform: i.platform, - type: i.type, - value: i.value, - verified: i.verified, - source: 'enrichment', - }, - true, - ) - } + didUpdate = true + await insertMemberIdentities( + qx, + squashedPayload.identities.map((i) => ({ + memberId, + platform: i.platform, + type: i.type, + value: i.value, + verified: i.verified, + source: 'enrichment', + })), + true, + ) } // process contributions diff --git a/services/apps/members_enrichment_worker/src/activities/member.ts b/services/apps/members_enrichment_worker/src/activities/member.ts index f34855c343..ccf30f34ff 100644 --- a/services/apps/members_enrichment_worker/src/activities/member.ts +++ b/services/apps/members_enrichment_worker/src/activities/member.ts @@ -2,8 +2,8 @@ import { CommonMemberService } from '@crowd/common_services' import { MemberField, PgPromiseQueryExecutor, - createMemberIdentity, findMemberById, + insertMemberIdentities, pgpQx, } from '@crowd/data-access-layer' import { @@ -88,15 +88,18 @@ export async function updateMemberWithEnrichmentData( attributes?: IAttributes, ): Promise { await svc.postgres.writer.connection().tx(async (tx) => { - for (const identity of identities) { - await createMemberIdentity(new PgPromiseQueryExecutor(tx), { - memberId, - platform: identity.platform, - value: identity.value, - type: identity.type, - verified: identity.verified || false, - source: 'enrichment', - }) + if (identities.length > 0) { + await insertMemberIdentities( + new PgPromiseQueryExecutor(tx), + identities.map((identity) => ({ + memberId, + platform: identity.platform, + value: identity.value, + type: identity.type, + verified: identity.verified || false, + source: 'enrichment', + })), + ) } if (attributes) { await updateMemberAttributes(tx, memberId, attributes) diff --git a/services/libs/common_services/src/services/member/unmerge.ts b/services/libs/common_services/src/services/member/unmerge.ts index fdba875892..73c3ae9878 100644 --- a/services/libs/common_services/src/services/member/unmerge.ts +++ b/services/libs/common_services/src/services/member/unmerge.ts @@ -10,7 +10,6 @@ import { addMemberRole, changeMemberOrganizationAffiliationOverrides, createMember, - createMemberIdentity, deleteManyMemberIdentities, fetchManyMemberOrgsWithOrgData, fetchManyOrganizationAffiliationPolicies, @@ -18,6 +17,7 @@ import { findAlreadyExistingVerifiedIdentities, findMemberById, findNonExistingOrganizationIds, + insertMemberIdentities, removeMemberRole, updateMember, } from '@crowd/data-access-layer' @@ -502,17 +502,20 @@ export async function unmergeMember( ) // Create identities for the secondary member - for (const i of secondaryIdentities) { - await createMemberIdentity(tx, { - memberId: secondaryId, - platform: i.platform, - type: i.type, - value: i.value, - sourceId: i.sourceId || null, - integrationId: i.integrationId || null, - verified: i.verified, - source: i.source, - }) + if (secondaryIdentities.length > 0) { + await insertMemberIdentities( + tx, + secondaryIdentities.map((i) => ({ + memberId: secondaryId, + platform: i.platform, + type: i.type, + value: i.value, + sourceId: i.sourceId || null, + integrationId: i.integrationId || null, + verified: i.verified, + source: i.source, + })), + ) } // Move affiliations diff --git a/services/libs/data-access-layer/src/member_segment_affiliations/index.ts b/services/libs/data-access-layer/src/member_segment_affiliations/index.ts index b8f0ecaf99..89686a15c2 100644 --- a/services/libs/data-access-layer/src/member_segment_affiliations/index.ts +++ b/services/libs/data-access-layer/src/member_segment_affiliations/index.ts @@ -118,7 +118,6 @@ export async function insertMemberSegmentAffiliations( affiliations.map((a) => ({ ...a, id: a.id ?? generateUUIDv1(), - // NOT NULL column — must set explicitly while listed in INSERT, else undefined → NULL verified: a.verified ?? false, })), failOnConflict ? undefined : 'DO NOTHING', diff --git a/services/libs/data-access-layer/src/members/identities.ts b/services/libs/data-access-layer/src/members/identities.ts index d7d47f4939..0e1b05e33f 100644 --- a/services/libs/data-access-layer/src/members/identities.ts +++ b/services/libs/data-access-layer/src/members/identities.ts @@ -272,17 +272,9 @@ export async function insertMemberIdentities( 'verifiedBy', ], identities.map((i) => ({ + ...i, id: i.id ?? generateUUIDv1(), - memberId: i.memberId, tenantId: DEFAULT_TENANT_ID, - integrationId: i.integrationId ?? null, - platform: i.platform, - source: i.source ?? null, - sourceId: i.sourceId ?? null, - value: i.value, - type: i.type, - verified: i.verified ?? true, - verifiedBy: i.verifiedBy ?? null, })), failOnConflict ? undefined : 'DO NOTHING', returnRows, @@ -295,32 +287,6 @@ export async function insertMemberIdentities( return qx.result(query) } -export async function createMemberIdentity( - qx: QueryExecutor, - i: MemberIdentityDbInsert, - failOnConflict: boolean, - returnRows: true, -): Promise -export async function createMemberIdentity( - qx: QueryExecutor, - i: MemberIdentityDbInsert, - failOnConflict?: boolean, - returnRows?: false, -): Promise -export async function createMemberIdentity( - qx: QueryExecutor, - i: MemberIdentityDbInsert, - failOnConflict = false, - returnRows = false, -): Promise { - if (returnRows) { - const rows = await insertMemberIdentities(qx, [i], failOnConflict, true) - return rows[0] - } - - await insertMemberIdentities(qx, [i], failOnConflict) -} - export async function moveToNewMember( qx: QueryExecutor, p: { diff --git a/services/libs/data-access-layer/src/members/organizations.ts b/services/libs/data-access-layer/src/members/organizations.ts index 8d846b9620..3a31a2a9d0 100644 --- a/services/libs/data-access-layer/src/members/organizations.ts +++ b/services/libs/data-access-layer/src/members/organizations.ts @@ -337,17 +337,13 @@ export async function insertMemberOrganizations( 'updatedAt', ], organizations.map((o) => ({ + ...o, id: o.id ?? generateUUIDv1(), - memberId: o.memberId, - organizationId: o.organizationId, - dateStart: o.dateStart ?? null, - dateEnd: o.dateEnd ?? null, - title: o.title ?? null, - source: o.source ?? null, - verified: o.verified ?? false, - verifiedBy: o.verifiedBy ?? null, + // NOT NULL, no DB default createdAt: ts, updatedAt: ts, + // NOT NULL DEFAULT false — must set while column is in INSERT list + verified: o.verified ?? false, })), failOnConflict ? undefined : 'DO NOTHING', returnRows, diff --git a/services/libs/data-access-layer/src/organizations/base.ts b/services/libs/data-access-layer/src/organizations/base.ts index cd766a7878..3d236d6407 100644 --- a/services/libs/data-access-layer/src/organizations/base.ts +++ b/services/libs/data-access-layer/src/organizations/base.ts @@ -418,30 +418,15 @@ export async function insertOrganizations( 'updatedAt', ], organizations.map((o) => ({ + ...o, id: o.id ?? generateUUIDv1(), - displayName: o.displayName ?? null, - description: o.description ?? null, - logo: o.logo ?? null, - tags: o.tags ?? null, - employees: o.employees ?? null, - revenueRange: o.revenueRange ?? null, - importHash: o.importHash ?? null, - location: o.location ?? null, - isTeamOrganization: o.isTeamOrganization ?? false, - isAffiliationBlocked: o.isAffiliationBlocked ?? false, - type: o.type ?? null, - size: o.size ?? null, - headline: o.headline ?? null, - industry: o.industry ?? null, - founded: o.founded ?? null, - employeeChurnRate: o.employeeChurnRate ?? null, - employeeGrowthRate: o.employeeGrowthRate ?? null, - manuallyCreated: o.manuallyCreated ?? false, - createdById: o.createdById ?? null, - updatedById: o.updatedById ?? null, tenantId: DEFAULT_TENANT_ID, createdAt: ts, updatedAt: ts, + // NOT NULL DEFAULT false — must set while column is in INSERT list + isTeamOrganization: o.isTeamOrganization ?? false, + isAffiliationBlocked: o.isAffiliationBlocked ?? false, + manuallyCreated: o.manuallyCreated ?? false, })), failOnConflict ? undefined : 'DO NOTHING', returnRows, diff --git a/services/libs/data-access-layer/src/organizations/identities.ts b/services/libs/data-access-layer/src/organizations/identities.ts index 13381bea6d..ad6d1f9c96 100644 --- a/services/libs/data-access-layer/src/organizations/identities.ts +++ b/services/libs/data-access-layer/src/organizations/identities.ts @@ -108,15 +108,8 @@ export async function insertOrganizationIdentities( 'integrationId', ], identities.map((i) => ({ - organizationId: i.organizationId, - platform: i.platform, - value: i.value, - type: i.type, - verified: i.verified, - source: i.source ?? null, - sourceId: i.sourceId ?? null, + ...i, tenantId: DEFAULT_TENANT_ID, - integrationId: i.integrationId ?? null, })), failOnConflict ? undefined : 'DO NOTHING', returnRows, diff --git a/services/libs/data-access-layer/src/segments/index.ts b/services/libs/data-access-layer/src/segments/index.ts index 7c8d52df4e..7fbd056a3d 100644 --- a/services/libs/data-access-layer/src/segments/index.ts +++ b/services/libs/data-access-layer/src/segments/index.ts @@ -190,8 +190,6 @@ export async function insertSegments( return returnRows ? [] : 0 } - const ts = new Date() - const query = prepareBulkInsert( 'segments', [ @@ -215,31 +213,14 @@ export async function insertSegments( // `type` is GENERATED ALWAYS from parentSlug/grandparentSlug — do not insert 'maturity', 'tenantId', - 'createdAt', - 'updatedAt', ], segments.map((s) => ({ + ...s, id: s.id ?? generateUUIDv1(), - slug: s.slug, - name: s.name ?? null, - url: s.url ?? null, + tenantId: DEFAULT_TENANT_ID, + // NOT NULL with defaults — must set while column is in INSERT list status: s.status ?? SegmentStatus.ACTIVE, isLF: s.isLF ?? true, - parentName: s.parentName ?? null, - grandparentName: s.grandparentName ?? null, - parentSlug: s.parentSlug ?? null, - grandparentSlug: s.grandparentSlug ?? null, - description: s.description ?? null, - sourceId: s.sourceId ?? null, - sourceParentId: s.sourceParentId ?? null, - customActivityTypes: s.customActivityTypes ?? {}, - activityChannels: s.activityChannels ?? {}, - parentId: s.parentId ?? null, - grandparentId: s.grandparentId ?? null, - maturity: s.maturity ?? null, - tenantId: DEFAULT_TENANT_ID, - createdAt: ts, - updatedAt: ts, })), failOnConflict ? undefined : 'DO NOTHING', returnRows, diff --git a/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts b/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts index 68976c168b..d1108a7647 100644 --- a/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts +++ b/services/libs/test-kit/src/factories/member-organization-affiliation-override.ts @@ -13,15 +13,5 @@ export async function upsertMemberOrganizationAffiliationOverrides( return [] } - return changeMemberOrganizationAffiliationOverrides( - qx, - data.map((row) => ({ - id: row.id, - memberId: row.memberId, - memberOrganizationId: row.memberOrganizationId, - allowAffiliation: row.allowAffiliation ?? undefined, - isPrimaryWorkExperience: row.isPrimaryWorkExperience ?? undefined, - })), - true, - ) + return changeMemberOrganizationAffiliationOverrides(qx, data, true) } diff --git a/services/libs/types/src/db/member-organization-affiliation-overrides.ts b/services/libs/types/src/db/member-organization-affiliation-overrides.ts index f5082f44f1..dec9385c2f 100644 --- a/services/libs/types/src/db/member-organization-affiliation-overrides.ts +++ b/services/libs/types/src/db/member-organization-affiliation-overrides.ts @@ -10,4 +10,11 @@ export type MemberOrganizationAffiliationOverrideDbInsert = Pick< MemberOrganizationAffiliationOverrideDbRow, 'memberId' | 'memberOrganizationId' > & - Partial> + Partial< + Pick & { + allowAffiliation: NonNullable + isPrimaryWorkExperience: NonNullable< + MemberOrganizationAffiliationOverrideDbRow['isPrimaryWorkExperience'] + > + } + > From 974ad43602b22d2366d44703540ded5dd04cba71 Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:06:09 +0530 Subject: [PATCH 6/9] refactor: collapse member segment affiliation inserts into one helper Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- .../patchProjectAffiliation.ts | 9 ++--- .../member/memberAffiliationsRepository.ts | 14 ++++++-- .../database/repositories/memberRepository.ts | 13 +++++-- .../src/member_segment_affiliations/index.ts | 17 --------- .../src/members/projectAffiliations.ts | 36 ------------------- 5 files changed, 27 insertions(+), 62 deletions(-) diff --git a/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts b/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts index b432b07058..edeae16954 100644 --- a/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts +++ b/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts @@ -10,7 +10,7 @@ import { fetchMemberSegmentAffiliationsForProject, findMaintainerRoles, findMemberById, - insertVerifiedMemberSegmentAffiliations, + insertMemberSegmentAffiliations, } from '@crowd/data-access-layer' import type { ISegmentAffiliationWithOrg } from '@crowd/data-access-layer' import { deleteMemberSegmentAffiliations } from '@crowd/data-access-layer/src/member_segment_affiliations' @@ -83,14 +83,15 @@ export async function patchProjectAffiliation(req: Request, res: Response): Prom await deleteMemberSegmentAffiliations(tx, { memberId, segmentId: projectId }) if (affiliations.length > 0) { - await insertVerifiedMemberSegmentAffiliations( + await insertMemberSegmentAffiliations( tx, - memberId, - projectId, affiliations.map((a) => ({ + memberId, + segmentId: projectId, organizationId: a.organizationId, dateStart: a.dateStart.toISOString(), dateEnd: a.dateEnd?.toISOString() ?? null, + verified: true, verifiedBy: verifiedBy!, })), ) diff --git a/backend/src/database/repositories/member/memberAffiliationsRepository.ts b/backend/src/database/repositories/member/memberAffiliationsRepository.ts index 7d4488e957..1759d0297b 100644 --- a/backend/src/database/repositories/member/memberAffiliationsRepository.ts +++ b/backend/src/database/repositories/member/memberAffiliationsRepository.ts @@ -2,7 +2,7 @@ import { OrganizationField, queryOrgs } from '@crowd/data-access-layer' import { deleteMemberSegmentAffiliations, fetchMemberAffiliations, - insertMemberAffiliations, + insertMemberSegmentAffiliations, } from '@crowd/data-access-layer/src/member_segment_affiliations' import { fetchManySegments } from '@crowd/data-access-layer/src/segments' import { IMemberAffiliation, IOrganization, SegmentData } from '@crowd/types' @@ -89,8 +89,16 @@ class MemberAffiliationsRepository { await deleteMemberSegmentAffiliations(qx, { memberId }) if (data?.length > 0) { - // Insert multiple member affiliations - await insertMemberAffiliations(qx, memberId, data) + await insertMemberSegmentAffiliations( + qx, + data.map((item) => ({ + memberId, + segmentId: item.segmentId, + organizationId: item.organizationId, + dateStart: item.dateStart || null, + dateEnd: item.dateEnd || null, + })), + ) } await SequelizeRepository.commitTransaction(transaction) diff --git a/backend/src/database/repositories/memberRepository.ts b/backend/src/database/repositories/memberRepository.ts index 84d3d6465f..d8b3831d1f 100644 --- a/backend/src/database/repositories/memberRepository.ts +++ b/backend/src/database/repositories/memberRepository.ts @@ -33,7 +33,7 @@ import { addMemberNoMerge, removeMemberToMerge } from '@crowd/data-access-layer/ import { deleteMemberSegmentAffiliations, findMemberAffiliations, - insertMemberAffiliations, + insertMemberSegmentAffiliations, } from '@crowd/data-access-layer/src/member_segment_affiliations' import { MemberField, @@ -1219,7 +1219,16 @@ class MemberRepository { return } - await insertMemberAffiliations(qx, memberId, data) + await insertMemberSegmentAffiliations( + qx, + data.map((item) => ({ + memberId, + segmentId: item.segmentId, + organizationId: item.organizationId, + dateStart: item.dateStart || null, + dateEnd: item.dateEnd || null, + })), + ) }), ) } diff --git a/services/libs/data-access-layer/src/member_segment_affiliations/index.ts b/services/libs/data-access-layer/src/member_segment_affiliations/index.ts index 89686a15c2..91ed33a4af 100644 --- a/services/libs/data-access-layer/src/member_segment_affiliations/index.ts +++ b/services/libs/data-access-layer/src/member_segment_affiliations/index.ts @@ -131,23 +131,6 @@ export async function insertMemberSegmentAffiliations( return qx.result(query) } -/** @deprecated Prefer `insertMemberSegmentAffiliations` with full insert payloads. */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export async function insertMemberAffiliations(qx: QueryExecutor, memberId: string, data: any[]) { - return insertMemberSegmentAffiliations( - qx, - data.map((item) => ({ - memberId, - segmentId: item.segmentId, - organizationId: item.organizationId, - dateStart: item.dateStart || null, - dateEnd: item.dateEnd || null, - })), - false, - false, - ) -} - export async function fetchMemberAffiliations( qx: QueryExecutor, memberId: string, diff --git a/services/libs/data-access-layer/src/members/projectAffiliations.ts b/services/libs/data-access-layer/src/members/projectAffiliations.ts index 5e472d1299..92e629001c 100644 --- a/services/libs/data-access-layer/src/members/projectAffiliations.ts +++ b/services/libs/data-access-layer/src/members/projectAffiliations.ts @@ -121,42 +121,6 @@ export async function fetchMemberSegmentAffiliationsForProject( ) } -export interface ISegmentAffiliationInsert { - organizationId: string - dateStart: string | null - dateEnd: string | null - verifiedBy: string -} - -/** - * Insert multiple verified segment affiliations for a member + project (segment) combination. - */ -export async function insertVerifiedMemberSegmentAffiliations( - qx: QueryExecutor, - memberId: string, - segmentId: string, - affiliations: ISegmentAffiliationInsert[], -): Promise { - for (const aff of affiliations) { - await qx.result( - ` - INSERT INTO "memberSegmentAffiliations" - (id, "memberId", "segmentId", "organizationId", "dateStart", "dateEnd", verified, "verifiedBy") - VALUES - (gen_random_uuid(), $(memberId), $(segmentId), $(organizationId), $(dateStart), $(dateEnd), true, $(verifiedBy)) - `, - { - memberId, - segmentId, - organizationId: aff.organizationId, - dateStart: aff.dateStart, - dateEnd: aff.dateEnd, - verifiedBy: aff.verifiedBy, - }, - ) - } -} - /** * Fetch work experiences for a member with organization details. * Used as fallback affiliations when no segment affiliations exist for a project. From d2b3ab400d3a65faa795d6330dda9407077cfc76 Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:14:31 +0530 Subject: [PATCH 7/9] fix: ensure default value for verified field in upsertOrgIdentities Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- .../libs/data-access-layer/src/organizations/identities.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/libs/data-access-layer/src/organizations/identities.ts b/services/libs/data-access-layer/src/organizations/identities.ts index ad6d1f9c96..456318fb74 100644 --- a/services/libs/data-access-layer/src/organizations/identities.ts +++ b/services/libs/data-access-layer/src/organizations/identities.ts @@ -142,12 +142,13 @@ export async function upsertOrgIdentities( platform: i.platform, type: i.type, value: i.value, - verified: i.verified, + // NOT NULL DEFAULT false — undefined would insert NULL + verified: i.verified ?? false, source: i.source ?? null, sourceId: i.sourceId ?? null, integrationId: integrationId ?? null, }) - } else if (existing.verified !== i.verified) { + } else if (i.verified !== undefined && existing.verified !== i.verified) { toUpdate.push(i) } } From 3913399329a33119352f97d50529b01b69b4f439 Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:32:28 +0530 Subject: [PATCH 8/9] fix: add default value for verified field in member identity and affiliation inserts Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- .../v1/members/project-affiliations/patchProjectAffiliation.ts | 1 + .../repositories/member/memberAffiliationsRepository.ts | 1 + backend/src/database/repositories/memberRepository.ts | 3 ++- backend/src/services/member/memberIdentityService.ts | 2 +- services/libs/types/src/db/member-identities.ts | 3 ++- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts b/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts index edeae16954..dae531bff1 100644 --- a/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts +++ b/backend/src/api/public/v1/members/project-affiliations/patchProjectAffiliation.ts @@ -94,6 +94,7 @@ export async function patchProjectAffiliation(req: Request, res: Response): Prom verified: true, verifiedBy: verifiedBy!, })), + true, ) } }) diff --git a/backend/src/database/repositories/member/memberAffiliationsRepository.ts b/backend/src/database/repositories/member/memberAffiliationsRepository.ts index 1759d0297b..81a47c421f 100644 --- a/backend/src/database/repositories/member/memberAffiliationsRepository.ts +++ b/backend/src/database/repositories/member/memberAffiliationsRepository.ts @@ -98,6 +98,7 @@ class MemberAffiliationsRepository { dateStart: item.dateStart || null, dateEnd: item.dateEnd || null, })), + true, ) } diff --git a/backend/src/database/repositories/memberRepository.ts b/backend/src/database/repositories/memberRepository.ts index d8b3831d1f..8d4af4f24a 100644 --- a/backend/src/database/repositories/memberRepository.ts +++ b/backend/src/database/repositories/memberRepository.ts @@ -18,11 +18,11 @@ import { import { BotDetectionService, CommonMemberService } from '@crowd/common_services' import { OrganizationField, - insertMemberIdentities, deleteMemberIdentities, deleteMemberIdentitiesByCombinations, findAlreadyExistingVerifiedIdentities, getLastActivitiesForMembers, + insertMemberIdentities, queryActivityRelations, queryOrgs, updateVerifiedFlag, @@ -1228,6 +1228,7 @@ class MemberRepository { dateStart: item.dateStart || null, dateEnd: item.dateEnd || null, })), + true, ) }), ) diff --git a/backend/src/services/member/memberIdentityService.ts b/backend/src/services/member/memberIdentityService.ts index 7e6801ee46..0aa85205aa 100644 --- a/backend/src/services/member/memberIdentityService.ts +++ b/backend/src/services/member/memberIdentityService.ts @@ -3,7 +3,7 @@ import lodash from 'lodash' import { captureApiChange, memberEditIdentitiesAction } from '@crowd/audit-logs' import { Error404, Error409 } from '@crowd/common' -import { insertMemberIdentities, findIdentitiesForMembers } from '@crowd/data-access-layer' +import { findIdentitiesForMembers, insertMemberIdentities } from '@crowd/data-access-layer' import { deleteMemberIdentity, fetchMemberIdentities, diff --git a/services/libs/types/src/db/member-identities.ts b/services/libs/types/src/db/member-identities.ts index a744249372..5590e4bad4 100644 --- a/services/libs/types/src/db/member-identities.ts +++ b/services/libs/types/src/db/member-identities.ts @@ -19,7 +19,7 @@ export interface MemberIdentityDbRow { export type MemberIdentityDbInsert = Pick< MemberIdentityDbRow, - 'memberId' | 'platform' | 'value' | 'type' + 'memberId' | 'platform' | 'value' | 'type' | 'verified' > & Partial< Omit< @@ -28,6 +28,7 @@ export type MemberIdentityDbInsert = Pick< | 'platform' | 'value' | 'type' + | 'verified' | 'tenantId' | 'createdAt' | 'updatedAt' From 541e3a8244635f6459ed95e5b68b5b0a2e1aa02c Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Mon, 13 Jul 2026 21:52:29 +0530 Subject: [PATCH 9/9] fix: resolve pr review comments Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- docs/adr/0007-test-factory-primitives-and-defaults.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/adr/0007-test-factory-primitives-and-defaults.md b/docs/adr/0007-test-factory-primitives-and-defaults.md index 948f9d6553..9e9de8336b 100644 --- a/docs/adr/0007-test-factory-primitives-and-defaults.md +++ b/docs/adr/0007-test-factory-primitives-and-defaults.md @@ -12,7 +12,9 @@ We need a simple, durable rule for how test data is created — what the factory ## Decision -Treat test setup as **composition of sharp primitives**, not smart world-builders. Keep three layers separate. +Treat test setup as composition of sharp primitives, not smart world-builders. Primitives persist what the caller provides via the DAL; defaults are opt-in and allowlisted; tests own scenario identity and graph shape. + +## Conventions ### 1. Primitives @@ -41,8 +43,6 @@ Defaults do **not** build a realistic entity or graph. They only patch allowlist - File-local seed helpers only after real repetition; promote into the shared test kit only when a second suite needs the same helper. - Do not grow suite-specific “build the whole world” APIs in the shared kit. -This stays aligned with ADR-0006: factories may widen partial write payloads and fill defaults; they do not redefine row or DAL contracts. - ## Alternatives Considered ### Alternative 1: Always-on smart factories that scaffold graphs