chore(go): audit transaction engine
Share change filtering between the durable and test transaction paths, collapse repeated precondition handling, unify journal sequence parsing, and split the subprocess worker test file. Claude-Session: https://claude.ai/code/session_01Agoc9bRVE9KL7mmWtey9Vq
This commit is contained in:
1 parent
b085947547
commit
c1caed3d42
4 files changed
+328
-348
No files matched your search
@@ -5,9 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type renameFunc func(source, destination string) error
|
type renameFunc func(source, destination string) error
|
||||||
@@ -165,84 +163,60 @@ func writeChangesWithFileOps(
|
|||||||
if ops.moveNoReplace == nil {
|
if ops.moveNoReplace == nil {
|
||||||
return fmt.Errorf("%s is missing atomic file operations", label)
|
return fmt.Errorf("%s is missing atomic file operations", label)
|
||||||
}
|
}
|
||||||
filtered := make([]fileChange, 0, len(changes))
|
filtered, err := filterDurableChanges(changes, label)
|
||||||
targets := map[string]bool{}
|
if err != nil {
|
||||||
for _, change := range changes {
|
return err
|
||||||
if change == nil || (change.existed && !change.remove &&
|
|
||||||
bytes.Equal(change.original, change.next) &&
|
|
||||||
sameManagedFileMode(change.originalMode, change.mode)) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
target := filepath.Clean(change.filePath)
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
target = strings.ToLower(target)
|
|
||||||
}
|
|
||||||
if targets[target] {
|
|
||||||
return fmt.Errorf("%s contains duplicate file target %s", label, change.filePath)
|
|
||||||
}
|
|
||||||
targets[target] = true
|
|
||||||
filtered = append(filtered, *change)
|
|
||||||
}
|
}
|
||||||
if err := assertFilePreconditions(preconditions, label); err != nil {
|
checkPreconditions := func() error {
|
||||||
return fmt.Errorf("%s: %w", label, err)
|
if err := assertFilePreconditions(preconditions, label); err != nil {
|
||||||
|
return fmt.Errorf("%s: %w", label, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := checkPreconditions(); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
if len(filtered) == 0 {
|
if len(filtered) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
staged := make([]stagedChange, 0, len(filtered))
|
staged := make([]stagedChange, 0, len(filtered))
|
||||||
|
abortStaging := func(cause error) error {
|
||||||
|
return joinTransactionFailure(cause, cleanupStaging(staged), "cleanup failed")
|
||||||
|
}
|
||||||
for _, change := range filtered {
|
for _, change := range filtered {
|
||||||
if err := assertFilePreconditions(preconditions, label); err != nil {
|
if err := checkPreconditions(); err != nil {
|
||||||
cleanupErrors := cleanupStaging(staged)
|
return abortStaging(err)
|
||||||
return joinTransactionFailure(
|
|
||||||
fmt.Errorf("%s: %w", label, err),
|
|
||||||
cleanupErrors,
|
|
||||||
"cleanup failed",
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
item, err := stageChange(change)
|
item, err := stageChange(change)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cleanupErrors := cleanupStaging(staged)
|
return abortStaging(fmt.Errorf("%s: %w", label, err))
|
||||||
return joinTransactionFailure(fmt.Errorf("%s: %w", label, err), cleanupErrors, "cleanup failed")
|
|
||||||
}
|
}
|
||||||
staged = append(staged, item)
|
staged = append(staged, item)
|
||||||
if err := assertFilePreconditions(preconditions, label); err != nil {
|
|
||||||
cleanupErrors := cleanupStaging(staged)
|
|
||||||
return joinTransactionFailure(
|
|
||||||
fmt.Errorf("%s: %w", label, err),
|
|
||||||
cleanupErrors,
|
|
||||||
"cleanup failed",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if err := assertFilePreconditions(preconditions, label); err != nil {
|
if err := checkPreconditions(); err != nil {
|
||||||
cleanupErrors := cleanupStaging(staged)
|
return abortStaging(err)
|
||||||
return joinTransactionFailure(fmt.Errorf("%s: %w", label, err), cleanupErrors, "cleanup failed")
|
}
|
||||||
|
recoverStaged := func(cause error) error {
|
||||||
|
failures := rollbackChanges(staged, ops)
|
||||||
|
failures = append(failures, cleanupStaging(staged)...)
|
||||||
|
return joinTransactionFailure(cause, failures, "recovery failed")
|
||||||
}
|
}
|
||||||
for index := range staged {
|
for index := range staged {
|
||||||
if err := assertFilePreconditions(preconditions, label); err != nil {
|
if err := checkPreconditions(); err != nil {
|
||||||
recoveryErrors := rollbackChanges(staged, ops)
|
return recoverStaged(err)
|
||||||
recoveryErrors = append(recoveryErrors, cleanupStaging(staged)...)
|
|
||||||
return joinTransactionFailure(fmt.Errorf("%s: %w", label, err), recoveryErrors, "recovery failed")
|
|
||||||
}
|
}
|
||||||
if err := commitChange(&staged[index], ops); err != nil {
|
if err := commitChange(&staged[index], ops); err != nil {
|
||||||
recoveryErrors := rollbackChanges(staged, ops)
|
return recoverStaged(fmt.Errorf("%s: %w", label, err))
|
||||||
recoveryErrors = append(recoveryErrors, cleanupStaging(staged)...)
|
|
||||||
return joinTransactionFailure(fmt.Errorf("%s: %w", label, err), recoveryErrors, "recovery failed")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := assertFilePreconditions(preconditions, label); err != nil {
|
if err := checkPreconditions(); err != nil {
|
||||||
recoveryErrors := rollbackChanges(staged, ops)
|
return recoverStaged(err)
|
||||||
recoveryErrors = append(recoveryErrors, cleanupStaging(staged)...)
|
|
||||||
return joinTransactionFailure(fmt.Errorf("%s: %w", label, err), recoveryErrors, "recovery failed")
|
|
||||||
}
|
}
|
||||||
if err := assertCommittedChanges(staged); err != nil {
|
if err := assertCommittedChanges(staged); err != nil {
|
||||||
recoveryErrors := rollbackChanges(staged, ops)
|
return recoverStaged(fmt.Errorf("%s: %w", label, err))
|
||||||
recoveryErrors = append(recoveryErrors, cleanupStaging(staged)...)
|
|
||||||
return joinTransactionFailure(fmt.Errorf("%s: %w", label, err), recoveryErrors, "recovery failed")
|
|
||||||
}
|
}
|
||||||
cleanupErrors := cleanupStaging(staged)
|
if failures := cleanupStaging(staged); len(failures) > 0 {
|
||||||
if len(cleanupErrors) > 0 {
|
return joinTransactionFailure(fmt.Errorf("%s cleanup failed", label), failures, "cleanup failed")
|
||||||
return joinTransactionFailure(fmt.Errorf("%s cleanup failed", label), cleanupErrors, "cleanup failed")
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -178,8 +178,7 @@ func loadDurableJournal(directory string) (*durableJournal, error) {
|
|||||||
}
|
}
|
||||||
stateNames := make([]string, 0)
|
stateNames := make([]string, 0)
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
if !entry.IsDir() && strings.HasPrefix(entry.Name(), "state-") &&
|
if _, state := durableJournalStateSequence(entry.Name()); state && !entry.IsDir() {
|
||||||
strings.HasSuffix(entry.Name(), ".json") {
|
|
||||||
stateNames = append(stateNames, entry.Name())
|
stateNames = append(stateNames, entry.Name())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -317,11 +316,11 @@ func validateDurableJournalProgress(journal *durableJournal) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func durableJournalStateSequence(name string) (uint64, bool) {
|
func durableJournalSequence(name, suffix string) (uint64, bool) {
|
||||||
if !strings.HasPrefix(name, "state-") || !strings.HasSuffix(name, ".json") {
|
if !strings.HasPrefix(name, "state-") || !strings.HasSuffix(name, suffix) {
|
||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
value := strings.TrimSuffix(strings.TrimPrefix(name, "state-"), ".json")
|
value := strings.TrimSuffix(strings.TrimPrefix(name, "state-"), suffix)
|
||||||
if len(value) != 20 {
|
if len(value) != 20 {
|
||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
@@ -329,14 +328,10 @@ func durableJournalStateSequence(name string) (uint64, bool) {
|
|||||||
return sequence, err == nil
|
return sequence, err == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func durableJournalStateSequence(name string) (uint64, bool) {
|
||||||
|
return durableJournalSequence(name, ".json")
|
||||||
|
}
|
||||||
|
|
||||||
func durableJournalPendingSequence(name string) (uint64, bool) {
|
func durableJournalPendingSequence(name string) (uint64, bool) {
|
||||||
if !strings.HasPrefix(name, "state-") || !strings.HasSuffix(name, ".pending") {
|
return durableJournalSequence(name, ".pending")
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
value := strings.TrimSuffix(strings.TrimPrefix(name, "state-"), ".pending")
|
|
||||||
if len(value) != 20 {
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
sequence, err := strconv.ParseUint(value, 10, 64)
|
|
||||||
return sequence, err == nil
|
|
||||||
}
|
}
|
||||||
@@ -13,282 +13,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
const durableWorkerEnvironment = "ELYDORA_DURABLE_TRANSACTION_WORKER"
|
|
||||||
|
|
||||||
func TestDurableTransactionSubprocessWorker(t *testing.T) {
|
|
||||||
mode := os.Getenv(durableWorkerEnvironment)
|
|
||||||
if mode == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
target := os.Getenv("ELYDORA_DURABLE_TARGET")
|
|
||||||
switch mode {
|
|
||||||
case "crash-update":
|
|
||||||
runDurableCrashWorker(t, target)
|
|
||||||
case "crash-multi-update":
|
|
||||||
runDurableMultiCrashWorker(t, target, os.Getenv("ELYDORA_DURABLE_SECOND_TARGET"))
|
|
||||||
case "crash-create":
|
|
||||||
runDurableCreateCrashWorker(t, target)
|
|
||||||
case "crash-delete":
|
|
||||||
runDurableDeleteCrashWorker(t, target)
|
|
||||||
case "crash-committed-cleanup":
|
|
||||||
runDurableCommittedCleanupCrashWorker(t, target)
|
|
||||||
case "crash-rollback-hardlink":
|
|
||||||
runDurableRollbackHardlinkCrashWorker(t)
|
|
||||||
case "crash-init-workspace-empty":
|
|
||||||
runDurableInitializationWorkspaceCrashWorker(t, target, false)
|
|
||||||
case "crash-init-workspace-owner":
|
|
||||||
runDurableInitializationWorkspaceCrashWorker(t, target, true)
|
|
||||||
case "crash-init-stage-full":
|
|
||||||
runDurableInitializationStageCrashWorker(t, target, false)
|
|
||||||
case "crash-init-stage-partial":
|
|
||||||
runDurableInitializationStageCrashWorker(t, target, true)
|
|
||||||
case "crash-init-probe":
|
|
||||||
runDurableInitializationProbeCrashWorker(t, target)
|
|
||||||
case "crash-journal-owner":
|
|
||||||
runDurableJournalOwnerCrashWorker(t)
|
|
||||||
case "crash-journal-pending-full":
|
|
||||||
runDurableJournalPendingCrashWorker(t, target, false)
|
|
||||||
case "crash-journal-pending-partial":
|
|
||||||
runDurableJournalPendingCrashWorker(t, target, true)
|
|
||||||
case "crash-workspace-marker-cleanup":
|
|
||||||
runDurableWorkspaceMarkerCleanupCrashWorker(t, target)
|
|
||||||
case "crash-journal-marker-cleanup":
|
|
||||||
runDurableJournalMarkerCleanupCrashWorker(t, target)
|
|
||||||
case "crash-torn-journal-tail":
|
|
||||||
runDurableTornJournalTailCrashWorker(t, target)
|
|
||||||
case "reader":
|
|
||||||
runDurableReaderWorker(t, target)
|
|
||||||
case "writer":
|
|
||||||
runDurableWriterWorker(t, target)
|
|
||||||
default:
|
|
||||||
t.Fatalf("unknown durable transaction worker %q", mode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runDurableRollbackHardlinkCrashWorker(t *testing.T) {
|
|
||||||
stateRoot, err := durableTransactionStateRoot()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
entries, err := os.ReadDir(stateRoot)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
journalPath := ""
|
|
||||||
for _, entry := range entries {
|
|
||||||
if entry.IsDir() && strings.HasPrefix(entry.Name(), "txn-") {
|
|
||||||
if journalPath != "" {
|
|
||||||
t.Fatal("multiple recovery journals found")
|
|
||||||
}
|
|
||||||
journalPath = filepath.Join(stateRoot, entry.Name())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if journalPath == "" {
|
|
||||||
t.Fatal("recovery journal is unavailable")
|
|
||||||
}
|
|
||||||
journal, err := loadDurableJournal(journalPath)
|
|
||||||
if err != nil || len(journal.Entries) != 1 {
|
|
||||||
t.Fatalf("load recovery journal: %v", err)
|
|
||||||
}
|
|
||||||
entry := journal.Entries[0]
|
|
||||||
if err := os.Link(entry.Path, entry.DiscardPath); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := syncTransactionDirectory(filepath.Dir(entry.DiscardPath)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
os.Exit(91)
|
|
||||||
}
|
|
||||||
|
|
||||||
func runDurableMultiCrashWorker(t *testing.T, firstTarget, secondTarget string) {
|
|
||||||
changes := make([]*fileChange, 0, 2)
|
|
||||||
for _, item := range []struct {
|
|
||||||
path string
|
|
||||||
value string
|
|
||||||
}{{firstTarget, "first-next"}, {secondTarget, "second-next"}} {
|
|
||||||
change, err := prepareFileChange(item.path, "multi crash target", []byte(item.value), 0600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
changes = append(changes, change)
|
|
||||||
}
|
|
||||||
ops := newTransactionFileOps(nil)
|
|
||||||
replace := ops.replaceWithBackup
|
|
||||||
commits := 0
|
|
||||||
ops.replaceWithBackup = func(current, replacement, backup string) error {
|
|
||||||
if err := replace(current, replacement, backup); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
commits++
|
|
||||||
if commits == 2 {
|
|
||||||
os.Exit(87)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := writeChangesWithFileOps(changes, "multi crash transaction", ops); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Fatal("multi crash worker completed without terminating")
|
|
||||||
}
|
|
||||||
|
|
||||||
func runDurableCreateCrashWorker(t *testing.T, target string) {
|
|
||||||
change, err := prepareFileChange(target, "crash create target", []byte("created"), 0600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ops := newTransactionFileOps(nil)
|
|
||||||
move := ops.moveNoReplace
|
|
||||||
ops.moveNoReplace = func(source, destination string) error {
|
|
||||||
if err := move(source, destination); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if sameTransactionTestPath(destination, target) {
|
|
||||||
os.Exit(88)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := writeChangesWithFileOps([]*fileChange{change}, "crash create", ops); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Fatal("create crash worker completed without terminating")
|
|
||||||
}
|
|
||||||
|
|
||||||
func runDurableDeleteCrashWorker(t *testing.T, target string) {
|
|
||||||
change, err := prepareSourceChange(
|
|
||||||
target,
|
|
||||||
"crash delete target",
|
|
||||||
[]byte("delete-original"),
|
|
||||||
true,
|
|
||||||
nil,
|
|
||||||
0600,
|
|
||||||
true,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ops := newTransactionFileOps(nil)
|
|
||||||
move := ops.moveNoReplace
|
|
||||||
ops.moveNoReplace = func(source, destination string) error {
|
|
||||||
if err := move(source, destination); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if sameTransactionTestPath(source, target) {
|
|
||||||
os.Exit(89)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := writeChangesWithFileOps([]*fileChange{change}, "crash delete", ops); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Fatal("delete crash worker completed without terminating")
|
|
||||||
}
|
|
||||||
|
|
||||||
func runDurableCommittedCleanupCrashWorker(t *testing.T, target string) {
|
|
||||||
change, err := prepareFileChange(target, "committed cleanup target", []byte("committed"), 0600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ops := newTransactionFileOps(nil)
|
|
||||||
replace := ops.replaceWithBackup
|
|
||||||
blocker := ""
|
|
||||||
ops.replaceWithBackup = func(current, replacement, backup string) error {
|
|
||||||
if err := replace(current, replacement, backup); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
blocker = filepath.Join(filepath.Dir(backup), "cleanup-blocker")
|
|
||||||
return os.WriteFile(blocker, []byte("block"), 0600)
|
|
||||||
}
|
|
||||||
err = writeChangesWithFileOps([]*fileChange{change}, "committed cleanup", ops)
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "committed; durable cleanup failed") {
|
|
||||||
t.Fatalf("committed cleanup error = %v", err)
|
|
||||||
}
|
|
||||||
if err := os.Remove(blocker); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
os.Exit(90)
|
|
||||||
}
|
|
||||||
|
|
||||||
func runDurableCrashWorker(t *testing.T, target string) {
|
|
||||||
change, err := prepareFileChange(target, "crash target", []byte("next"), 0600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ops := newTransactionFileOps(nil)
|
|
||||||
replace := ops.replaceWithBackup
|
|
||||||
absoluteTarget, err := filepath.Abs(target)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ops.replaceWithBackup = func(current, replacement, backup string) error {
|
|
||||||
if err := replace(current, replacement, backup); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if sameTransactionTestPath(current, absoluteTarget) {
|
|
||||||
os.Exit(86)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := writeChangesWithFileOps([]*fileChange{change}, "crash transaction", ops); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Fatal("crash worker completed without terminating")
|
|
||||||
}
|
|
||||||
|
|
||||||
func runDurableReaderWorker(t *testing.T, target string) {
|
|
||||||
stop := os.Getenv("ELYDORA_DURABLE_STOP")
|
|
||||||
for {
|
|
||||||
if _, err := os.Lstat(stop); err == nil {
|
|
||||||
return
|
|
||||||
} else if !errors.Is(err, os.ErrNotExist) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
content, err := os.ReadFile(target)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
t.Fatalf("reader observed missing target: %v", err)
|
|
||||||
}
|
|
||||||
if _, statErr := os.Lstat(target); statErr != nil {
|
|
||||||
t.Fatalf("reader observed unavailable target: %v; stat=%v", err, statErr)
|
|
||||||
}
|
|
||||||
time.Sleep(time.Millisecond)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if string(content) != "value-a" && string(content) != "value-b" {
|
|
||||||
t.Fatalf("reader observed partial value %q", content)
|
|
||||||
}
|
|
||||||
time.Sleep(time.Millisecond)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func runDurableWriterWorker(t *testing.T, target string) {
|
|
||||||
value := os.Getenv("ELYDORA_DURABLE_VALUE")
|
|
||||||
ready := os.Getenv("ELYDORA_DURABLE_READY")
|
|
||||||
start := os.Getenv("ELYDORA_DURABLE_START")
|
|
||||||
change, err := prepareFileChange(target, "writer target", []byte(value), 0600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(ready, []byte("ready"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
deadline := time.Now().Add(15 * time.Second)
|
|
||||||
for {
|
|
||||||
if _, err := os.Lstat(start); err == nil {
|
|
||||||
break
|
|
||||||
} else if !errors.Is(err, os.ErrNotExist) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatal("writer start barrier timed out")
|
|
||||||
}
|
|
||||||
time.Sleep(time.Millisecond)
|
|
||||||
}
|
|
||||||
if err := writeChanges([]*fileChange{change}, "subprocess writer", nil); err != nil {
|
|
||||||
fmt.Fprintln(os.Stderr, err)
|
|
||||||
os.Exit(23)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDurableTransactionRecoversAfterProcessTermination(t *testing.T) {
|
func TestDurableTransactionRecoversAfterProcessTermination(t *testing.T) {
|
||||||
home := filepath.Join(t.TempDir(), "home")
|
home := filepath.Join(t.TempDir(), "home")
|
||||||
if err := createPrivateTransactionDirectory(home); err != nil {
|
if err := createPrivateTransactionDirectory(home); err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,287 @@
|
|||||||
|
package plugins
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const durableWorkerEnvironment = "ELYDORA_DURABLE_TRANSACTION_WORKER"
|
||||||
|
|
||||||
|
func TestDurableTransactionSubprocessWorker(t *testing.T) {
|
||||||
|
mode := os.Getenv(durableWorkerEnvironment)
|
||||||
|
if mode == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
target := os.Getenv("ELYDORA_DURABLE_TARGET")
|
||||||
|
switch mode {
|
||||||
|
case "crash-update":
|
||||||
|
runDurableCrashWorker(t, target)
|
||||||
|
case "crash-multi-update":
|
||||||
|
runDurableMultiCrashWorker(t, target, os.Getenv("ELYDORA_DURABLE_SECOND_TARGET"))
|
||||||
|
case "crash-create":
|
||||||
|
runDurableCreateCrashWorker(t, target)
|
||||||
|
case "crash-delete":
|
||||||
|
runDurableDeleteCrashWorker(t, target)
|
||||||
|
case "crash-committed-cleanup":
|
||||||
|
runDurableCommittedCleanupCrashWorker(t, target)
|
||||||
|
case "crash-rollback-hardlink":
|
||||||
|
runDurableRollbackHardlinkCrashWorker(t)
|
||||||
|
case "crash-init-workspace-empty":
|
||||||
|
runDurableInitializationWorkspaceCrashWorker(t, target, false)
|
||||||
|
case "crash-init-workspace-owner":
|
||||||
|
runDurableInitializationWorkspaceCrashWorker(t, target, true)
|
||||||
|
case "crash-init-stage-full":
|
||||||
|
runDurableInitializationStageCrashWorker(t, target, false)
|
||||||
|
case "crash-init-stage-partial":
|
||||||
|
runDurableInitializationStageCrashWorker(t, target, true)
|
||||||
|
case "crash-init-probe":
|
||||||
|
runDurableInitializationProbeCrashWorker(t, target)
|
||||||
|
case "crash-journal-owner":
|
||||||
|
runDurableJournalOwnerCrashWorker(t)
|
||||||
|
case "crash-journal-pending-full":
|
||||||
|
runDurableJournalPendingCrashWorker(t, target, false)
|
||||||
|
case "crash-journal-pending-partial":
|
||||||
|
runDurableJournalPendingCrashWorker(t, target, true)
|
||||||
|
case "crash-workspace-marker-cleanup":
|
||||||
|
runDurableWorkspaceMarkerCleanupCrashWorker(t, target)
|
||||||
|
case "crash-journal-marker-cleanup":
|
||||||
|
runDurableJournalMarkerCleanupCrashWorker(t, target)
|
||||||
|
case "crash-torn-journal-tail":
|
||||||
|
runDurableTornJournalTailCrashWorker(t, target)
|
||||||
|
case "reader":
|
||||||
|
runDurableReaderWorker(t, target)
|
||||||
|
case "writer":
|
||||||
|
runDurableWriterWorker(t, target)
|
||||||
|
default:
|
||||||
|
t.Fatalf("unknown durable transaction worker %q", mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDurableRollbackHardlinkCrashWorker(t *testing.T) {
|
||||||
|
stateRoot, err := durableTransactionStateRoot()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(stateRoot)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
journalPath := ""
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.IsDir() && strings.HasPrefix(entry.Name(), "txn-") {
|
||||||
|
if journalPath != "" {
|
||||||
|
t.Fatal("multiple recovery journals found")
|
||||||
|
}
|
||||||
|
journalPath = filepath.Join(stateRoot, entry.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if journalPath == "" {
|
||||||
|
t.Fatal("recovery journal is unavailable")
|
||||||
|
}
|
||||||
|
journal, err := loadDurableJournal(journalPath)
|
||||||
|
if err != nil || len(journal.Entries) != 1 {
|
||||||
|
t.Fatalf("load recovery journal: %v", err)
|
||||||
|
}
|
||||||
|
entry := journal.Entries[0]
|
||||||
|
if err := os.Link(entry.Path, entry.DiscardPath); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := syncTransactionDirectory(filepath.Dir(entry.DiscardPath)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
os.Exit(91)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDurableMultiCrashWorker(t *testing.T, firstTarget, secondTarget string) {
|
||||||
|
changes := make([]*fileChange, 0, 2)
|
||||||
|
for _, item := range []struct {
|
||||||
|
path string
|
||||||
|
value string
|
||||||
|
}{{firstTarget, "first-next"}, {secondTarget, "second-next"}} {
|
||||||
|
change, err := prepareFileChange(item.path, "multi crash target", []byte(item.value), 0600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
changes = append(changes, change)
|
||||||
|
}
|
||||||
|
ops := newTransactionFileOps(nil)
|
||||||
|
replace := ops.replaceWithBackup
|
||||||
|
commits := 0
|
||||||
|
ops.replaceWithBackup = func(current, replacement, backup string) error {
|
||||||
|
if err := replace(current, replacement, backup); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
commits++
|
||||||
|
if commits == 2 {
|
||||||
|
os.Exit(87)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := writeChangesWithFileOps(changes, "multi crash transaction", ops); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Fatal("multi crash worker completed without terminating")
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDurableCreateCrashWorker(t *testing.T, target string) {
|
||||||
|
change, err := prepareFileChange(target, "crash create target", []byte("created"), 0600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ops := newTransactionFileOps(nil)
|
||||||
|
move := ops.moveNoReplace
|
||||||
|
ops.moveNoReplace = func(source, destination string) error {
|
||||||
|
if err := move(source, destination); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if sameTransactionTestPath(destination, target) {
|
||||||
|
os.Exit(88)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := writeChangesWithFileOps([]*fileChange{change}, "crash create", ops); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Fatal("create crash worker completed without terminating")
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDurableDeleteCrashWorker(t *testing.T, target string) {
|
||||||
|
change, err := prepareSourceChange(
|
||||||
|
target,
|
||||||
|
"crash delete target",
|
||||||
|
[]byte("delete-original"),
|
||||||
|
true,
|
||||||
|
nil,
|
||||||
|
0600,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ops := newTransactionFileOps(nil)
|
||||||
|
move := ops.moveNoReplace
|
||||||
|
ops.moveNoReplace = func(source, destination string) error {
|
||||||
|
if err := move(source, destination); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if sameTransactionTestPath(source, target) {
|
||||||
|
os.Exit(89)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := writeChangesWithFileOps([]*fileChange{change}, "crash delete", ops); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Fatal("delete crash worker completed without terminating")
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDurableCommittedCleanupCrashWorker(t *testing.T, target string) {
|
||||||
|
change, err := prepareFileChange(target, "committed cleanup target", []byte("committed"), 0600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ops := newTransactionFileOps(nil)
|
||||||
|
replace := ops.replaceWithBackup
|
||||||
|
blocker := ""
|
||||||
|
ops.replaceWithBackup = func(current, replacement, backup string) error {
|
||||||
|
if err := replace(current, replacement, backup); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
blocker = filepath.Join(filepath.Dir(backup), "cleanup-blocker")
|
||||||
|
return os.WriteFile(blocker, []byte("block"), 0600)
|
||||||
|
}
|
||||||
|
err = writeChangesWithFileOps([]*fileChange{change}, "committed cleanup", ops)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "committed; durable cleanup failed") {
|
||||||
|
t.Fatalf("committed cleanup error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(blocker); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
os.Exit(90)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDurableCrashWorker(t *testing.T, target string) {
|
||||||
|
change, err := prepareFileChange(target, "crash target", []byte("next"), 0600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ops := newTransactionFileOps(nil)
|
||||||
|
replace := ops.replaceWithBackup
|
||||||
|
absoluteTarget, err := filepath.Abs(target)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ops.replaceWithBackup = func(current, replacement, backup string) error {
|
||||||
|
if err := replace(current, replacement, backup); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if sameTransactionTestPath(current, absoluteTarget) {
|
||||||
|
os.Exit(86)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := writeChangesWithFileOps([]*fileChange{change}, "crash transaction", ops); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Fatal("crash worker completed without terminating")
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDurableReaderWorker(t *testing.T, target string) {
|
||||||
|
stop := os.Getenv("ELYDORA_DURABLE_STOP")
|
||||||
|
for {
|
||||||
|
if _, err := os.Lstat(stop); err == nil {
|
||||||
|
return
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
content, err := os.ReadFile(target)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("reader observed missing target: %v", err)
|
||||||
|
}
|
||||||
|
if _, statErr := os.Lstat(target); statErr != nil {
|
||||||
|
t.Fatalf("reader observed unavailable target: %v; stat=%v", err, statErr)
|
||||||
|
}
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if string(content) != "value-a" && string(content) != "value-b" {
|
||||||
|
t.Fatalf("reader observed partial value %q", content)
|
||||||
|
}
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDurableWriterWorker(t *testing.T, target string) {
|
||||||
|
value := os.Getenv("ELYDORA_DURABLE_VALUE")
|
||||||
|
ready := os.Getenv("ELYDORA_DURABLE_READY")
|
||||||
|
start := os.Getenv("ELYDORA_DURABLE_START")
|
||||||
|
change, err := prepareFileChange(target, "writer target", []byte(value), 0600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(ready, []byte("ready"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(15 * time.Second)
|
||||||
|
for {
|
||||||
|
if _, err := os.Lstat(start); err == nil {
|
||||||
|
break
|
||||||
|
} else if !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("writer start barrier timed out")
|
||||||
|
}
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
}
|
||||||
|
if err := writeChanges([]*fileChange{change}, "subprocess writer", nil); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(23)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user