Store Records

To store records in SecretVault, you'll first encrypt any sensitive fields with nilQL, then store the record across your cluster's nodes. Each node stores plaintext fields identically, while encrypted fields are split into unique shares, one per node.

1. Check Your Collection for the Schema ID

List available Collections using the List Schemas endpoint (GET /schemas) to get the Schema ID of the collection you want to store your data in.

2. Store a Record

Before storing:

• Encrypt any private fields using nilQL to get encrypted shares
• Make sure data matches your Collection's JSON Schema
• Generate and include a unique _id (UUIDv4) for each record

note

You can include up to 10k records in a POST /data/create request, and the total JSON body size can be up to 16MB

Example POST /data/create Payload

{
  "schema": "9b22147f-d6d5-40f1-927d-96c08XXXXXXXX",
  "data": [
    {
      "_id": "120a60f3-d1b6-4a11-a413-abcd60c8ccb3",
      "years_in_web3": "zKH1L", // encrypted share
      "responses": [
        { "rating": 5, "question_number": 1 },
        { "rating": 3, "question_number": 2 }
      ]
    },
    {
      "_id": "b321301b-ffcd-43d8-bec7-4b0830f7a305",
      "years_in_web3": "5J_KL4", // encrypted share
      "responses": [
        { "rating": 2, "question_number": 1 },
        { "rating": 4, "question_number": 2 }
      ]
    }
  ]
}

Upload to all nodes using each node's Upload/Create Data endpoint using valid API tokens to authenticate requests to each node. The response includes:

• Created records
• Any validation errors, where any issues with data ingestion will be described even if the communication with the node resulted in a 200 status response. You can check out an example below that showcases both instances:

Example POST /data/create Response

{
   "data": {
      "created": [
         "120a60f3-d1b6-4a11-a413-abcd60c8ccb3"
      ],
      "errors": [
         {
            "error": "E11000 duplicate key error collection: datablocks_data.f47ac10b-58cc-4372-a567-0e02b2XXXXXX index: _id_ dup key: { _id: UUID(\"b321301b-ffcd-43d8-bec7-4b0830f7a305\") }",
            "document": {
              "_id": "b321301b-ffcd-43d8-bec7-4b0830f7a305",
              "years_in_web3": "5J_KL4",
              "responses": [
                { "rating": 2, "question_number": 1 },
                { "rating": 4, "question_number": 2 }
              ]
            }
         }
      ]
   }
}

Code Samples

Python

nildb/secretvault_python/nildb_api.py

def data_upload(self, node_name: str, schema_id: str, payload: list) -> bool:
    """Create/upload records in the specified node and schema."""
    try:
        node = self.nodes[node_name]
        headers = {
            'Authorization': f'Bearer {node["jwt"]}',
            'Content-Type': 'application/json'
        }
        
        body = {
            "schema": schema_id,
            "data": payload
        }

        response = requests.post(
            f"{node['url']}/api/v1/data/create",
            headers=headers,
            json=body
        )
        
        return response.status_code == 200 and response.json().get("data", {}).get("errors", []) == []
    except Exception as e:
        print(f"Error creating records in {node_name}: {str(e)}")
        return False

View on GitHub

TypeScript

nildb/secretvault_nextjs/app/lib/nildb.ts

export const createNilDBAPI = (config = NODE_CONFIG) => {
  const uploadCredential = async (
    nodeName: NodeName,
    credentialData: CredentialPayload
  ): Promise<boolean> => {
    const node = config[nodeName];

    try {
      const response = await fetch(`${node.url}/api/v1/data/create`, {
        method: 'POST',
        headers: {
          Authorization: `Bearer ${node.jwt}`,
          'Content-Type': 'application/json',
        },
        body: JSON.stringify({
          schema: credentialData.schema,
          data: [credentialData.data],
        }),
      });

      if (!response.ok) {
        throw new Error(`HTTP error! status: ${response.status}`);
      }

      return true;
    } catch (error) {
      console.error(`Error creating credential in ${String(nodeName)}:`, error);
      return false;
    }

View on GitHub

JavaScript (with wrapper)

Install secretvaults

npm i secretvaults

Run the readWriteSv script

node readWriteSv.js

readWriteSv.js

examples/store/dataCreateRead.js

import { SecretVaultWrapper } from "secretvaults";
import { orgConfig } from "../orgConfig.js";

// update schema id with your own value
const SCHEMA_ID = "d8cbedef-e12a-468e-b5cf-caba3172afad";

// '%allot' signals that the value will be encrypted to one %share per node before writing to the collection
const web3ExperienceSurveyData = [
  {
    years_in_web3: { "%allot": 4 },
    responses: [
      { rating: 5, question_number: 1 },
      { rating: 3, question_number: 2 },
    ],
  },
  {
    years_in_web3: { "%allot": 1 },
    responses: [
      { rating: 2, question_number: 1 },
      { rating: 4, question_number: 2 },
    ],
  },
];

async function main() {
  try {
    const collection = new SecretVaultWrapper(
      orgConfig.nodes,
      orgConfig.orgCredentials,
      SCHEMA_ID,
    );
    await collection.init();

    const dataWritten = await collection.writeToNodes(web3ExperienceSurveyData);
    console.log("dataWritten", dataWritten);

    const newIds = [
      ...new Set(dataWritten.flatMap((item) => item.data.created)),
    ];
    console.log("created ids:", newIds);

    const dataRead = await collection.readFromNodes({});
    console.log("📚 total records:", dataRead.length);
    console.log(
      "📚 Read new records:",
      dataRead.slice(0, web3ExperienceSurveyData.length),
    );
  } catch (error) {
    console.error("❌ Failed to use SecretVaultWrapper:", error.message);
    process.exit(1);
  }
}

main();

View on GitHub

orgConfig.js

examples/orgConfig.js

import dotenv from "dotenv";
dotenv.config();

export const orgConfig = {
  orgCredentials: {
    secretKey: process.env.NILLION_ORG_SECRET_KEY,
    orgDid: process.env.NILLION_ORG_DID,
  },
  nodes: [
    {
      url: "https://nildb-nx8v.nillion.network",
      did: "did:nil:testnet:nillion1qfrl8nje3nvwh6cryj63mz2y6gsdptvn07nx8v",
    },
    {
      url: "https://nildb-p3mx.nillion.network",
      did: "did:nil:testnet:nillion1uak7fgsp69kzfhdd6lfqv69fnzh3lprg2mp3mx",
    },
    {
      url: "https://nildb-rugk.nillion.network",
      did: "did:nil:testnet:nillion1kfremrp2mryxrynx66etjl8s7wazxc3rssrugk",
    },
  ],
};

View on GitHub

Python (with wrapper)

Install secretvaults

pip install secretvaults

Run the data_create_read script

python3 data_create_read.py

data_create_read.py

examples/store_encryption/data_create_read.py

"""Data Create and Read example using the SecretVault wrapper"""

import asyncio
import json
import random
import sys

from secretvaults import SecretVaultWrapper, OperationType
from org_config import org_config

# Update schema ID with your own value
SCHEMA_ID = "51dba4eb-b5e7-4c54-9059-867ff592d1ae"


# %allot signals that the value will be encrypted to one %share per node before writing to the collection
def generate_web3_experience_survey_data(num_entries=10):
    data = []

    for _ in range(num_entries):
        entry = {
            "years_in_web3": {"%allot": random.randint(1, 10)},
            "responses": [
                {
                    "rating": random.randint(1, 5),
                    "question_number": random.randint(1, 10),
                }
                for _ in range(random.randint(3, 10))
            ],
        }
        data.append(entry)

    return data


# Generate some entries
web3_experience_survey_data = generate_web3_experience_survey_data(10)


async def main():
    """
    Main function to demonstrate writing to and reading from nodes using the SecretVaultWrapper.
    """
    try:
        # Initialize the SecretVaultWrapper instance with the org configuration and schema ID
        collection = SecretVaultWrapper(
            org_config["nodes"],
            org_config["org_credentials"],
            SCHEMA_ID,
            operation=OperationType.STORE,
        )
        await collection.init()

        # Write data to nodes
        data_written = await collection.write_to_nodes(web3_experience_survey_data)

        # Extract unique created IDs from the results
        new_ids = list(
            {
                created_id
                for item in data_written
                if item.get("result")
                for created_id in item["result"]["data"]["created"]
            }
        )
        print("🔏 Created IDs:")
        print("\n".join(new_ids))

        # Read data from nodes
        data_read = await collection.read_from_nodes()
        print("📚 Read new records:", (json.dumps(data_read[: len(web3_experience_survey_data)], indent=2)))

    except RuntimeError as error:
        print(f"❌ Failed to use SecretVaultWrapper: {str(error)}")
        sys.exit(1)


# Run the async main function
if __name__ == "__main__":
    asyncio.run(main())

View on GitHub

org_config.py

examples/org_config.py

"""The SecretVault organization configuration"""

import os
from dotenv import load_dotenv

load_dotenv()

# Organization configuration
org_config = {
    "org_credentials": {
        "secret_key": os.getenv("NILLION_ORG_SECRET_KEY"),
        "org_did": os.getenv("NILLION_ORG_DID"),
    },
    "nodes": [
        {
            "url": "https://nildb-nx8v.nillion.network",
            "did": "did:nil:testnet:nillion1qfrl8nje3nvwh6cryj63mz2y6gsdptvn07nx8v",
        },
        {
            "url": "https://nildb-p3mx.nillion.network",
            "did": "did:nil:testnet:nillion1uak7fgsp69kzfhdd6lfqv69fnzh3lprg2mp3mx",
        },
        {
            "url": "https://nildb-rugk.nillion.network",
            "did": "did:nil:testnet:nillion1kfremrp2mryxrynx66etjl8s7wazxc3rssrugk",
        },
    ],
}

View on GitHub