Query and Data Invalidation Architecture

fetch(`${API}/datasets?search=${search}`)
  .then(r => r.json())
  .then(data => setDatasets(data));

const { data: datasets = [] } = useSearchDatasets(search, true, projectId);

queryClient.invalidateQueries({ queryKey: ['processes'] });
// or
queryClient.refetchQueries({ queryKey: ['processes'] });

const { invalidateProject } = useContext(ProcessContext);
await invalidateProject(projectId);

// Fetch all projects
const { data: projects = [], isLoading, error } = useProjects();

// Fetch all environments
const { data: environments = [], isLoading, error } = useEnvironments();

// Fetch process types for an environment
const { data: types = {}, isLoading } = useEnvironmentProcessTypes(environmentId);

// Fetch all processes for a project
const { data: processes = [], isLoading, error, refetch } = useProcesses(projectId);
// Note: refetch is available but prefer using invalidation helpers

// Fetch a single dataset by ID
const { data: dataset, isLoading, error } = useDataset(datasetId, projectId);

// Search datasets with filters
const { data: datasets = [], isLoading, error } = useSearchDatasets(
  searchText,      // Search string
  completedOnly,   // Boolean: only completed processes
  projectId        // Filter by project
);

// Fetch outputs for a specific process version
const { data: datasets = [], isLoading } = useProcessOutputDatasets(
  process,    // Process object
  version,    // Version number
  projectId   // Required - query is disabled without it
);

// Create a new project
const createProject = useCreateProject();
await createProject.mutateAsync({ name: "My Project" });

// Create a new environment
const createEnvironment = useCreateEnvironment();
await createEnvironment.mutateAsync({ name: "My Env", image: "..." });

// Create a new process (does NOT auto-invalidate)
const createProcess = useCreateProcess();
const newProcess = await createProcess.mutateAsync({
  proc: { name, type, params, ... },
  projectId
});
// Must manually invalidate after:
await invalidateProject(projectId);

// Cancel a queued/running process version (does NOT auto-invalidate)
const cancelProcess = useCancelProcess();
await cancelProcess.mutateAsync({ processId, version, projectId });
await invalidateProject(projectId);

import { useExportProject, useProjectExport, useImportProject, useProjectImport } from '../datamodel/useQueries';

// Start an export job (fire-and-forget)
const exportProject = useExportProject();
const { export_id: exportId } = await exportProject.mutateAsync(projectId);

// Poll it until done/failed - refetchInterval stops automatically at a terminal state
const { data: exportJob } = useProjectExport(projectId, exportId);
// exportJob.state: "pending" | "running" | "done" | "failed"

// Seed an already-created (empty) project from a previously-uploaded export zip
// (the zip must have been uploaded into that same project via uploadFile())
const importProject = useImportProject();
await importProject.mutateAsync({ projectId, uploadId });

// Poll the import job the same way
const { data: importJob } = useProjectImport(importId);

import { useAvailableStorageBackends, useAvailableClusters } from '../datamodel/useQueries';

// Storage backends available for new projects - not live/quota-based, normal staleTime
const { data: backends = [] } = useAvailableStorageBackends();

// Clusters available for a project, filtered by live resource limits (e.g. Kueue quota).
// No staleTime - refetches whenever projectId or resourceRequests changes the query key,
// since quota can change between requests.
const { data: clusters = [] } = useAvailableClusters(projectId, { cpu: "2", memory: "4Gi" });

import {
  useProjectMembers, useProjectInvites, useInviteMember, useCancelInvite,
  useLeaveProject, useInviteInfo, useAcceptInvite
} from '../datamodel/useQueries';

// List current members of a project
const { data: members = [] } = useProjectMembers(projectId);

// List pending invites for a project
const { data: invites = [] } = useProjectInvites(projectId);

// Invite a member by email (auto-invalidates projectInvites)
const inviteMember = useInviteMember(projectId);
await inviteMember.mutateAsync(email);

// Cancel a pending invite (auto-invalidates projectInvites)
const cancelInvite = useCancelInvite(projectId);
await cancelInvite.mutateAsync(inviteId);

// Leave a project (auto-invalidates the projects list)
const leaveProject = useLeaveProject(projectId);
await leaveProject.mutateAsync();

// Look up an invite by its token (e.g. on the accept-invite landing page); does not retry on error
const { data: invite, error } = useInviteInfo(token);

// Accept an invite (auto-invalidates the projects list)
const acceptInvite = useAcceptInvite();
await acceptInvite.mutateAsync(token);

import { usePublications, useCreatePublication, useDeletePublication } from '../datamodel/useQueries';

// List publications for a project
const { data: publications = [] } = usePublications(projectId);

// Create a publication (auto-invalidates publications)
const createPublication = useCreatePublication(projectId);
await createPublication.mutateAsync({ findable: false, allowAnonymous: true });

// Delete a publication (auto-invalidates publications)
const deletePublication = useDeletePublication(projectId);
await deletePublication.mutateAsync(publicationId);

import {
  useProjectTags, useCreateTag, useUpdateTag, useDeleteTag,
  useAddVersionTag, useRemoveVersionTag
} from '../datamodel/useQueries';

// List tags defined for a project
const { data: tags = [] } = useProjectTags(projectId);

// Create/update/delete a tag definition (all auto-invalidate projectTags)
const createTag = useCreateTag(projectId);
await createTag.mutateAsync({ name: "approved", color: "#22c55e" });

const updateTag = useUpdateTag(projectId);
await updateTag.mutateAsync({ tagId, tag: { name: "approved-v2" } });

const deleteTag = useDeleteTag(projectId);
await deleteTag.mutateAsync(tagId);

// Attach/remove a tag from a specific process version (does NOT auto-invalidate -
// callers must use ProcessContext invalidation helpers, since this changes process data)
const addVersionTag = useAddVersionTag();
await addVersionTag.mutateAsync({ processId, version, tagId, projectId });
await invalidateProcess(processId, projectId);

const removeVersionTag = useRemoveVersionTag();
await removeVersionTag.mutateAsync({ processId, version, tagId, projectId });
await invalidateProcess(processId, projectId);

import { usePlugins, useEnablePlugin, useDisablePlugin, useUpgradePlugin } from '../datamodel/useQueries';

// List all registered plugins (requires authentication)
const { data: plugins = [] } = usePlugins();

// Enable/disable/upgrade a plugin (all auto-invalidate the plugins list)
const enablePlugin = useEnablePlugin();
await enablePlugin.mutateAsync(pluginId);

const disablePlugin = useDisablePlugin();
await disablePlugin.mutateAsync(pluginId);

const upgradePlugin = useUpgradePlugin();
await upgradePlugin.mutateAsync(pluginId);

import {
  useWorkspaces, usePublicWorkspaces, useWorkspace, useSaveWorkspace,
  useSaveWorkspaceVersion, useUpdateWorkspace, useForkWorkspace, useDeleteWorkspace
} from '../datamodel/useQueries';

// List workspaces for a project
const { data: workspaces = [] } = useWorkspaces(projectId);

// List publicly-shared workspaces (no projectId scoping)
const { data: publicWorkspaces = [] } = usePublicWorkspaces();

// Fetch a single workspace by ID
const { data: workspace } = useWorkspace(workspaceId);

// Save a brand-new workspace (auto-invalidates workspaces for the project)
const saveWorkspace = useSaveWorkspace(projectId);
const newWorkspace = await saveWorkspace.mutateAsync({ title: "My Layout", layout });

// Save a new version onto an existing workspace (auto-invalidates workspaces for the project)
const saveWorkspaceVersion = useSaveWorkspaceVersion(projectId);
await saveWorkspaceVersion.mutateAsync({ workspaceId, layout });

// Update workspace metadata, e.g. title or public visibility
// (auto-invalidates both workspaces and publicWorkspaces)
const updateWorkspace = useUpdateWorkspace(projectId);
await updateWorkspace.mutateAsync({ workspaceId, title: "Renamed", is_public: true });

// Fork a workspace (optionally a specific version) into the current project
// (auto-invalidates workspaces for the project)
const forkWorkspace = useForkWorkspace(projectId);
await forkWorkspace.mutateAsync({ workspaceId, version });

// Delete a workspace (auto-invalidates workspaces for the project)
const deleteWorkspace = useDeleteWorkspace(projectId);
await deleteWorkspace.mutateAsync(workspaceId);

import { queryKeys } from './datamodel/useQueries';

queryKeys.projects                           // ['projects']
queryKeys.environments                       // ['environments']
queryKeys.environmentProcessTypes(envId)     // ['environmentProcessTypes', envId]
queryKeys.processes(projectId)               // ['processes', projectId]
queryKeys.dataset(id)                        // ['dataset', id]
queryKeys.datasets(search, completedOnly, projectId)  // ['datasets', { ... }]
queryKeys.processOutputDatasets(processId, version)   // ['processOutputDatasets', processId, version]
queryKeys.availableClusters(projectId, resourceRequests)  // ['availableClusters', projectId, cpu, memory]
queryKeys.availableStorageBackends           // ['availableStorageBackends']
queryKeys.projectExport(projectId, exportId) // ['projectExport', projectId, exportId]
queryKeys.projectImport(importId)            // ['projectImport', importId]
queryKeys.projectMembers(projectId)          // ['projectMembers', projectId]
queryKeys.projectInvites(projectId)          // ['projectInvites', projectId]
queryKeys.publications(projectId)            // ['publications', projectId]
queryKeys.inviteInfo(token)                  // ['inviteInfo', token]
queryKeys.projectTags(projectId)             // ['projectTags', projectId]
queryKeys.workspaces(projectId)              // ['workspaces', projectId]
queryKeys.publicWorkspaces                   // ['publicWorkspaces']
queryKeys.workspace(id)                      // ['workspace', id]

const {
  invalidateProcess,
  invalidateProject,
  invalidateDatasets
} = useContext(ProcessContext);

// Example: After updating process parameters
await createProcess.mutateAsync({ proc: { id: processId, ... }, projectId });
await invalidateProcess(processId, projectId);

// Example: After creating a new process
const newProcess = await createProcess.mutateAsync({ proc, projectId });
await invalidateProject(projectId);
setActiveProcess({ processId: newProcess.id, version: 1 });

// Example: After dataset-only operation
await someDatasetOperation();
await invalidateDatasets();

const handleWebSocketMessage = useCallback(async (update) => {
  await invalidateHelpers.invalidateProject();
}, [invalidateHelpers]);

useWebSocket(`${WS_API}/ws/processes/updates`, {
  enabled: !!currentProject,
  name: 'Process State Updates',
  onMessage: handleWebSocketMessage
});

import { useContext } from 'react';
import { ProcessContext } from '../ProcessContext';
import { useCreateProcess } from '../datamodel/useQueries';

function MyComponent() {
  const { currentProject, invalidateProject, setActiveProcess } = useContext(ProcessContext);
  const createProcess = useCreateProcess();

  const handleCreate = async () => {
    // 1. Create the process
    const newProcess = await createProcess.mutateAsync({
      proc: {
        name: "my-process",
        type: "fft",
        params: { ... },
        environment_id: envId
      },
      projectId: currentProject
    });

    // 2. Invalidate to update all components
    await invalidateProject(currentProject);

    // 3. Navigate to the new process
    setActiveProcess({
      processId: newProcess.id,
      version: 1
    });
  };

  return <button onClick={handleCreate}>Create Process</button>;
}

import { useState, useEffect, useRef } from 'react';
import { useSearchDatasets } from '../datamodel/useQueries';

function DatasetSearch({ projectId }) {
  const [searchText, setSearchText] = useState('');
  const [debouncedSearch, setDebouncedSearch] = useState('');
  const debounceTimer = useRef(null);

  // Debounce search input
  useEffect(() => {
    if (debounceTimer.current) clearTimeout(debounceTimer.current);
    debounceTimer.current = setTimeout(() => {
      setDebouncedSearch(searchText);
    }, 300);
    return () => {
      if (debounceTimer.current) clearTimeout(debounceTimer.current);
    };
  }, [searchText]);

  // Use the hook with debounced search
  const { data: datasets = [], isLoading } = useSearchDatasets(
    debouncedSearch,
    true,  // completedOnly
    projectId
  );

  return (
    <div>
      <input value={searchText} onChange={e => setSearchText(e.target.value)} />
      {isLoading ? <div>Loading...</div> : (
        <ul>
          {datasets.map(ds => <li key={ds.id}>{ds.dataset_name}</li>)}
        </ul>
      )}
    </div>
  );
}

import { useContext } from 'react';
import { ProcessContext } from '../ProcessContext';
import { useProcessOutputDatasets } from '../datamodel/useQueries';

function ProcessOutputs() {
  const { currentProject, processes, activeProcess } = useContext(ProcessContext);

  // Find the active process object
  const process = activeProcess
    ? processes.find(p => p.id === activeProcess.processId)
    : null;

  // Fetch outputs for active version
  const { data: datasets = [], isLoading } = useProcessOutputDatasets(
    process,
    activeProcess?.version,
    currentProject
  );

  if (!process) return <div>No process selected</div>;
  if (isLoading) return <div>Loading outputs...</div>;

  return (
    <ul>
      {datasets.map(ds => (
        <li key={ds.id}>{ds.dataset_name}: {ds.url}</li>
      ))}
    </ul>
  );
}

import { useContext } from 'react';
import { ProcessContext } from '../ProcessContext';
import { useCreateProcess } from '../datamodel/useQueries';

function ProcessParameterEditor({ process }) {
  const { currentProject, invalidateProject, setActiveProcess } = useContext(ProcessContext);
  const createProcess = useCreateProcess();

  const handleSave = async (newParams) => {
    // Creating with existing ID creates a new version
    const updatedProcess = await createProcess.mutateAsync({
      proc: {
        id: process.id,  // Same ID = new version
        name: process.name,
        type: process.type,
        environment_id: process.environment_id,
        params: newParams
      },
      projectId: currentProject
    });

    // Invalidate to show the new version
    await invalidateProject(currentProject);

    // Switch to the new version
    const latestVersion = Math.max(...updatedProcess.versions.map(v => v.version));
    setActiveProcess({
      processId: process.id,
      version: latestVersion
    });
  };

  return <ParameterForm onSave={handleSave} />;
}

// ✅ Good: Wait for refetch to complete
await invalidateProject(projectId);
setActiveProcess({ processId: newId, version: 1 });

// ❌ Bad: Race condition - activeProcess set before data arrives
invalidateProject(projectId);  // Don't await
setActiveProcess({ processId: newId, version: 1 });

// ✅ Good: Only fetch when needed
const { data: datasets } = useProcessOutputDatasets(
  process,
  version,
  projectId,
  { enabled: !!process && version != null && !!projectId }
);

// ❌ Bad: Missing projectId - query stays disabled (enabled checks !!projectId internally)
const { data: datasets } = useProcessOutputDatasets(process, version);

// ✅ Good: Prevents undefined errors
const { data: processes = [] } = useProcesses(projectId);
processes.map(p => ...);  // Safe even during loading

// ❌ Bad: Errors during loading
const { data: processes } = useProcesses(projectId);
processes.map(p => ...);  // Error: Cannot read property 'map' of undefined

// ❌ Bad: Component won't update when cache invalidates
useEffect(() => {
  fetch(`${API}/datasets?search=${search}`)
    .then(r => r.json())
    .then(setDatasets);
}, [search]);

// ✅ Good: Automatically updates
const { data: datasets = [] } = useSearchDatasets(search, true, projectId);

// ❌ Bad: UI won't update
const newProcess = await createProcess.mutateAsync({ proc, projectId });
setActiveProcess({ processId: newProcess.id, version: 1 });
// Where's the invalidation?

// ✅ Good: UI updates immediately
const newProcess = await createProcess.mutateAsync({ proc, projectId });
await invalidateProject(projectId);
setActiveProcess({ processId: newProcess.id, version: 1 });

// ❌ Overkill: Invalidates everything when only one process changed
await invalidateProject(projectId);

// ✅ Better: Only invalidate what changed
await invalidateProcess(processId, projectId);

// ❌ Bad: Race condition
invalidateProject(projectId);
console.log('Processes:', processes);  // Still old data!

// ✅ Good: Wait for refetch
await invalidateProject(projectId);
console.log('Processes:', processes);  // New data available

// ❌ Bad: Causes infinite loop
function MyComponent() {
  const { invalidateProject } = useContext(ProcessContext);
  invalidateProject();  // Called every render!
  return <div>...</div>;
}

// ✅ Good: Invalidate in effect or handler
function MyComponent() {
  const { invalidateProject } = useContext(ProcessContext);

  useEffect(() => {
    invalidateProject();
  }, []);  // Only once

  return <div>...</div>;
}

// ❌ Bad: Mutation already invalidates, this is redundant
const createProject = useCreateProject();  // This auto-invalidates
await createProject.mutateAsync({ name });
await invalidateProjects();  // Unnecessary!

// Note: useCreateProcess does NOT auto-invalidate (by design)
// You must manually invalidate for processes
const createProcess = useCreateProcess();  // This does NOT auto-invalidate
await createProcess.mutateAsync({ proc, projectId });
await invalidateProject(projectId);  // Required!

import { ReactQueryDevtools } from '@tanstack/react-query-devtools';

function App() {
  return (
    <>
      {/* Your app */}
      <ReactQueryDevtools initialIsOpen={false} />
    </>
  );
}

const { data, isLoading, isFetching, isStale } = useProcesses(projectId);

console.log({
  data,
  isLoading,    // True during first load
  isFetching,   // True during any fetch (including refetch)
  isStale       // True if data is stale (past staleTime)
});

import { useQueryClient } from '@tanstack/react-query';

const queryClient = useQueryClient();

// Get current cache state
const processesQuery = queryClient.getQueryState(['processes', projectId]);
console.log('Status:', processesQuery?.status);
console.log('Data:', processesQuery?.data);

// Get cached data directly
const cachedProcesses = queryClient.getQueryData(['processes', projectId]);
console.log('Cached processes:', cachedProcesses);

// See all queries in cache
const allQueries = queryClient.getQueryCache().getAll();
console.log('All queries:', allQueries.map(q => q.queryKey));

// In ProcessContext.jsx (temporary debugging)
invalidateProject: async (projectId = currentProject) => {
  console.log('[invalidateProject] Starting for project:', projectId);

  await Promise.all([
    queryClient.refetchQueries({
      queryKey: ['processes', projectId],
      type: 'active'
    }).then(() => console.log('[invalidateProject] Processes refetched')),

    queryClient.refetchQueries({
      queryKey: ['datasets'],
      type: 'active'
    }).then(() => console.log('[invalidateProject] Datasets refetched')),

    // ... etc
  ]);

  console.log('[invalidateProject] Complete');
}

┌─────────────────────────────────────────────────────────────┐
│                        Backend API                          │
└─────────────────────────────────────────────────────────────┘
                              ▲
                              │ HTTP + WebSocket
                              ▼
┌─────────────────────────────────────────────────────────────┐
│                  TanStack Query Cache                       │
│  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐        │
│  │  Processes  │  │  Datasets   │  │   Outputs   │        │
│  └─────────────┘  └─────────────┘  └─────────────┘        │
└─────────────────────────────────────────────────────────────┘
                              ▲
                              │
            ┌─────────────────┼─────────────────┐
            │                 │                 │
            ▼                 ▼                 ▼
      ┌──────────┐      ┌──────────┐     ┌──────────┐
      │ FlowView │      │ PlotView │     │ Export   │
      │          │      │          │     │          │
      │ uses     │      │ uses     │     │ uses     │
      │ hooks    │      │ hooks    │     │ hooks    │
      └──────────┘      └──────────┘     └──────────┘
            │                 │                 │
            └─────────────────┼─────────────────┘
                              │
                              ▼
                    ┌──────────────────┐
                    │ ProcessContext   │
                    │                  │
                    │ Invalidation     │
                    │ Helpers          │
                    │                  │
                    │ - invalidate     │
                    │   Process()      │
                    │ - invalidate     │
                    │   Project()      │
                    │ - invalidate     │
                    │   Datasets()     │
                    └──────────────────┘
                              ▲
                              │
                    WebSocket Updates
                    Mutations
                    User Actions

const [datasets, setDatasets] = useState([]);
const [loading, setLoading] = useState(false);

useEffect(() => {
  setLoading(true);
  fetch(`${API}/datasets?search=${search}&project_id=${projectId}`)
    .then(r => r.json())
    .then(data => {
      setDatasets(data);
      setLoading(false);
    });
}, [search, projectId]);

const { data: datasets = [], isLoading } = useSearchDatasets(
  search,
  true,
  projectId
);