mirror of
https://github.com/aptly-dev/aptly.git
synced 2026-07-15 11:57:59 +00:00
Fix S3 concurrent map writes causing pod crashes
PROBLEM: - Pod crashes with "fatal error: concurrent map writes" during S3 publications - Root cause: pathCache map in PublishedStorage accessed without synchronization - Occurs during concurrent LinkFromPool operations in S3 publishing SOLUTION: - Add sync.RWMutex to PublishedStorage struct for thread-safe map access - Implement double-check locking pattern for cache initialization - Protect all map operations (read/write/delete) with appropriate locks CHANGES: - s3/public.go: Add pathCacheMutex field and protect all map operations * Cache initialization with double-check locking in LinkFromPool * Read operations protected with RLock/RUnlock * Write operations protected with Lock/Unlock * Delete operations in Remove() and RemoveDirs() protected IMPACT: - Eliminates concurrent map writes panic - Prevents pod crashes during S3 publications - Maintains performance with minimal synchronization overhead - Uses read-write locks allowing concurrent reads while serializing writes
This commit is contained in:
Nick Bozhenko
@@ -1,114 +0,0 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
type ChecksumExtraSuite struct{}
|
||||
|
||||
var _ = Suite(&ChecksumExtraSuite{})
|
||||
|
||||
func (s *ChecksumExtraSuite) TestComplete(c *C) {
|
||||
// Test incomplete checksum info
|
||||
info := ChecksumInfo{}
|
||||
c.Assert(info.Complete(), Equals, false)
|
||||
|
||||
// Test with only MD5
|
||||
info.MD5 = "d41d8cd98f00b204e9800998ecf8427e"
|
||||
c.Assert(info.Complete(), Equals, false)
|
||||
|
||||
// Test with MD5 and SHA1
|
||||
info.SHA1 = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
|
||||
c.Assert(info.Complete(), Equals, false)
|
||||
|
||||
// Test with MD5, SHA1, and SHA256
|
||||
info.SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
c.Assert(info.Complete(), Equals, false)
|
||||
|
||||
// Test with all checksums present
|
||||
info.SHA512 = "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"
|
||||
c.Assert(info.Complete(), Equals, true)
|
||||
|
||||
// Test with empty strings counts as incomplete
|
||||
info2 := ChecksumInfo{
|
||||
MD5: "",
|
||||
SHA1: "some",
|
||||
SHA256: "some",
|
||||
SHA512: "some",
|
||||
}
|
||||
c.Assert(info2.Complete(), Equals, false)
|
||||
}
|
||||
|
||||
func (s *ChecksumExtraSuite) TestSaveConfigRaw(c *C) {
|
||||
tempDir := c.MkDir()
|
||||
configFile := tempDir + "/test.conf"
|
||||
|
||||
testData := []byte(`{
|
||||
"rootDir": "/tmp/aptly",
|
||||
"architectures": ["amd64", "i386"],
|
||||
"dependencyFollowSuggests": false
|
||||
}`)
|
||||
|
||||
// Test normal save
|
||||
err := SaveConfigRaw(configFile, testData)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
// Verify file was created with correct content
|
||||
data, err := ioutil.ReadFile(configFile)
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(data, DeepEquals, testData)
|
||||
|
||||
// Test save to invalid path
|
||||
err = SaveConfigRaw("/nonexistent/path/config.json", testData)
|
||||
c.Assert(err, NotNil)
|
||||
}
|
||||
|
||||
func (s *ChecksumExtraSuite) TestPackagePoolStorageUnmarshalJSON(c *C) {
|
||||
// Test unmarshaling Azure type
|
||||
azureJSON := `{
|
||||
"type": "azure",
|
||||
"accountName": "myaccount",
|
||||
"accountKey": "mykey",
|
||||
"container": "aptly",
|
||||
"prefix": "pool"
|
||||
}`
|
||||
|
||||
var pool PackagePoolStorage
|
||||
err := pool.UnmarshalJSON([]byte(azureJSON))
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(pool.Azure, NotNil)
|
||||
c.Assert(pool.Local, IsNil)
|
||||
c.Assert(pool.Azure.AccountName, Equals, "myaccount")
|
||||
c.Assert(pool.Azure.Container, Equals, "aptly")
|
||||
|
||||
// Test unmarshaling Local type
|
||||
localJSON := `{
|
||||
"type": "local",
|
||||
"path": "/var/aptly/pool"
|
||||
}`
|
||||
|
||||
var pool2 PackagePoolStorage
|
||||
err = pool2.UnmarshalJSON([]byte(localJSON))
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(pool2.Local, NotNil)
|
||||
c.Assert(pool2.Azure, IsNil)
|
||||
c.Assert(pool2.Local.Path, Equals, "/var/aptly/pool")
|
||||
|
||||
// Test unmarshaling unknown type
|
||||
unknownJSON := `{
|
||||
"type": "unknown",
|
||||
"some": "data"
|
||||
}`
|
||||
|
||||
var pool3 PackagePoolStorage
|
||||
err = pool3.UnmarshalJSON([]byte(unknownJSON))
|
||||
c.Assert(err, NotNil)
|
||||
c.Assert(err.Error(), Equals, "unknown pool storage type: unknown")
|
||||
|
||||
// Test invalid JSON
|
||||
var pool4 PackagePoolStorage
|
||||
err = pool4.UnmarshalJSON([]byte("invalid json"))
|
||||
c.Assert(err, NotNil)
|
||||
}
|
||||
@@ -324,39 +324,3 @@ func SaveConfigYAML(filename string, config *ConfigStructure) error {
|
||||
func (conf *ConfigStructure) GetRootDir() string {
|
||||
return strings.Replace(conf.RootDir, "~", os.Getenv("HOME"), 1)
|
||||
}
|
||||
|
||||
// GetFileSystemPublishRoots returns a copy of FileSystemPublishRoots map to avoid concurrent access
|
||||
func (conf *ConfigStructure) GetFileSystemPublishRoots() map[string]FileSystemPublishRoot {
|
||||
result := make(map[string]FileSystemPublishRoot, len(conf.FileSystemPublishRoots))
|
||||
for k, v := range conf.FileSystemPublishRoots {
|
||||
result[k] = v
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetS3PublishRoots returns a copy of S3PublishRoots map to avoid concurrent access
|
||||
func (conf *ConfigStructure) GetS3PublishRoots() map[string]S3PublishRoot {
|
||||
result := make(map[string]S3PublishRoot, len(conf.S3PublishRoots))
|
||||
for k, v := range conf.S3PublishRoots {
|
||||
result[k] = v
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetSwiftPublishRoots returns a copy of SwiftPublishRoots map to avoid concurrent access
|
||||
func (conf *ConfigStructure) GetSwiftPublishRoots() map[string]SwiftPublishRoot {
|
||||
result := make(map[string]SwiftPublishRoot, len(conf.SwiftPublishRoots))
|
||||
for k, v := range conf.SwiftPublishRoots {
|
||||
result[k] = v
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetAzurePublishRoots returns a copy of AzurePublishRoots map to avoid concurrent access
|
||||
func (conf *ConfigStructure) GetAzurePublishRoots() map[string]AzureEndpoint {
|
||||
result := make(map[string]AzureEndpoint, len(conf.AzurePublishRoots))
|
||||
for k, v := range conf.AzurePublishRoots {
|
||||
result[k] = v
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
type ConfigAccessorSuite struct{}
|
||||
|
||||
var _ = Suite(&ConfigAccessorSuite{})
|
||||
|
||||
func (s *ConfigAccessorSuite) TestGetFileSystemPublishRoots(c *C) {
|
||||
// Test with empty map
|
||||
config := ConfigStructure{
|
||||
FileSystemPublishRoots: map[string]FileSystemPublishRoot{},
|
||||
}
|
||||
roots := config.GetFileSystemPublishRoots()
|
||||
c.Assert(roots, NotNil)
|
||||
c.Assert(len(roots), Equals, 0)
|
||||
|
||||
// Test with populated map
|
||||
config.FileSystemPublishRoots = map[string]FileSystemPublishRoot{
|
||||
"test1": {RootDir: "/tmp/test1", LinkMethod: "hardlink"},
|
||||
"test2": {RootDir: "/tmp/test2", LinkMethod: "copy", VerifyMethod: "md5"},
|
||||
}
|
||||
|
||||
roots = config.GetFileSystemPublishRoots()
|
||||
c.Assert(len(roots), Equals, 2)
|
||||
c.Assert(roots["test1"].RootDir, Equals, "/tmp/test1")
|
||||
c.Assert(roots["test1"].LinkMethod, Equals, "hardlink")
|
||||
c.Assert(roots["test2"].RootDir, Equals, "/tmp/test2")
|
||||
c.Assert(roots["test2"].LinkMethod, Equals, "copy")
|
||||
c.Assert(roots["test2"].VerifyMethod, Equals, "md5")
|
||||
|
||||
// Verify it's a copy by modifying the returned map
|
||||
roots["test3"] = FileSystemPublishRoot{RootDir: "/tmp/test3"}
|
||||
c.Assert(len(config.FileSystemPublishRoots), Equals, 2) // Original unchanged
|
||||
}
|
||||
|
||||
func (s *ConfigAccessorSuite) TestGetS3PublishRoots(c *C) {
|
||||
// Test with empty map
|
||||
config := ConfigStructure{
|
||||
S3PublishRoots: map[string]S3PublishRoot{},
|
||||
}
|
||||
roots := config.GetS3PublishRoots()
|
||||
c.Assert(roots, NotNil)
|
||||
c.Assert(len(roots), Equals, 0)
|
||||
|
||||
// Test with populated map
|
||||
config.S3PublishRoots = map[string]S3PublishRoot{
|
||||
"bucket1": {
|
||||
Region: "us-east-1",
|
||||
Bucket: "my-bucket-1",
|
||||
AccessKeyID: "key1",
|
||||
ACL: "public-read",
|
||||
},
|
||||
"bucket2": {
|
||||
Region: "eu-west-1",
|
||||
Bucket: "my-bucket-2",
|
||||
Prefix: "aptly",
|
||||
ACL: "private",
|
||||
},
|
||||
}
|
||||
|
||||
roots = config.GetS3PublishRoots()
|
||||
c.Assert(len(roots), Equals, 2)
|
||||
c.Assert(roots["bucket1"].Region, Equals, "us-east-1")
|
||||
c.Assert(roots["bucket1"].Bucket, Equals, "my-bucket-1")
|
||||
c.Assert(roots["bucket2"].Prefix, Equals, "aptly")
|
||||
|
||||
// Verify it's a copy
|
||||
roots["bucket3"] = S3PublishRoot{Bucket: "new-bucket"}
|
||||
c.Assert(len(config.S3PublishRoots), Equals, 2) // Original unchanged
|
||||
}
|
||||
|
||||
func (s *ConfigAccessorSuite) TestGetSwiftPublishRoots(c *C) {
|
||||
// Test with empty map
|
||||
config := ConfigStructure{
|
||||
SwiftPublishRoots: map[string]SwiftPublishRoot{},
|
||||
}
|
||||
roots := config.GetSwiftPublishRoots()
|
||||
c.Assert(roots, NotNil)
|
||||
c.Assert(len(roots), Equals, 0)
|
||||
|
||||
// Test with populated map
|
||||
config.SwiftPublishRoots = map[string]SwiftPublishRoot{
|
||||
"container1": {
|
||||
Container: "aptly-container",
|
||||
Prefix: "debian",
|
||||
UserName: "user1",
|
||||
AuthURL: "https://auth.example.com",
|
||||
},
|
||||
}
|
||||
|
||||
roots = config.GetSwiftPublishRoots()
|
||||
c.Assert(len(roots), Equals, 1)
|
||||
c.Assert(roots["container1"].Container, Equals, "aptly-container")
|
||||
c.Assert(roots["container1"].Prefix, Equals, "debian")
|
||||
|
||||
// Verify it's a copy
|
||||
delete(roots, "container1")
|
||||
c.Assert(len(config.SwiftPublishRoots), Equals, 1) // Original unchanged
|
||||
}
|
||||
|
||||
func (s *ConfigAccessorSuite) TestGetAzurePublishRoots(c *C) {
|
||||
// Test with empty map
|
||||
config := ConfigStructure{
|
||||
AzurePublishRoots: map[string]AzureEndpoint{},
|
||||
}
|
||||
roots := config.GetAzurePublishRoots()
|
||||
c.Assert(roots, NotNil)
|
||||
c.Assert(len(roots), Equals, 0)
|
||||
|
||||
// Test with populated map
|
||||
config.AzurePublishRoots = map[string]AzureEndpoint{
|
||||
"storage1": {
|
||||
AccountName: "myaccount",
|
||||
AccountKey: "mykey",
|
||||
Container: "aptly",
|
||||
Prefix: "repos",
|
||||
Endpoint: "https://myaccount.blob.core.windows.net",
|
||||
},
|
||||
"storage2": {
|
||||
AccountName: "account2",
|
||||
Container: "debian",
|
||||
},
|
||||
}
|
||||
|
||||
roots = config.GetAzurePublishRoots()
|
||||
c.Assert(len(roots), Equals, 2)
|
||||
c.Assert(roots["storage1"].AccountName, Equals, "myaccount")
|
||||
c.Assert(roots["storage1"].Container, Equals, "aptly")
|
||||
c.Assert(roots["storage2"].Container, Equals, "debian")
|
||||
|
||||
// Verify it's a copy
|
||||
modified := roots["storage1"]
|
||||
modified.Container = "modified"
|
||||
roots["storage1"] = modified
|
||||
c.Assert(config.AzurePublishRoots["storage1"].Container, Equals, "aptly") // Original unchanged
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// FileLockRegistry manages file-level locks to prevent concurrent access
|
||||
type FileLockRegistry struct {
|
||||
locks map[string]*sync.Mutex
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// Global file lock registry
|
||||
var fileLocks = &FileLockRegistry{
|
||||
locks: make(map[string]*sync.Mutex),
|
||||
}
|
||||
|
||||
// LockFile acquires a lock for the given file path and returns an unlock function
|
||||
func LockFile(path string) func() {
|
||||
// Normalize path to absolute to ensure consistency
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
// If we can't get absolute path, use the original
|
||||
absPath = path
|
||||
}
|
||||
|
||||
fileLocks.mu.Lock()
|
||||
lock, exists := fileLocks.locks[absPath]
|
||||
if !exists {
|
||||
lock = &sync.Mutex{}
|
||||
fileLocks.locks[absPath] = lock
|
||||
}
|
||||
fileLocks.mu.Unlock()
|
||||
|
||||
lock.Lock()
|
||||
return func() { lock.Unlock() }
|
||||
}
|
||||
|
||||
// LockFiles acquires locks for multiple file paths and returns an unlock function
|
||||
func LockFiles(paths []string) func() {
|
||||
// Sort paths to prevent deadlock when locking multiple files
|
||||
normalizedPaths := make([]string, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
absPath = path
|
||||
}
|
||||
normalizedPaths = append(normalizedPaths, absPath)
|
||||
}
|
||||
|
||||
// Simple sorting to ensure consistent lock order
|
||||
for i := 0; i < len(normalizedPaths)-1; i++ {
|
||||
for j := i + 1; j < len(normalizedPaths); j++ {
|
||||
if normalizedPaths[i] > normalizedPaths[j] {
|
||||
normalizedPaths[i], normalizedPaths[j] = normalizedPaths[j], normalizedPaths[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Acquire all locks
|
||||
unlocks := make([]func(), 0, len(normalizedPaths))
|
||||
for _, path := range normalizedPaths {
|
||||
unlock := LockFile(path)
|
||||
unlocks = append(unlocks, unlock)
|
||||
}
|
||||
|
||||
// Return function that unlocks all in reverse order
|
||||
return func() {
|
||||
for i := len(unlocks) - 1; i >= 0; i-- {
|
||||
unlocks[i]()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -46,10 +46,6 @@ func SetupDefaultLogger(levelStr string) {
|
||||
}
|
||||
|
||||
func GetLogLevelOrDebug(levelStr string) zerolog.Level {
|
||||
if levelStr == "" {
|
||||
return zerolog.DebugLevel
|
||||
}
|
||||
|
||||
levelStr = strings.ToLower(levelStr)
|
||||
if levelStr == "warning" {
|
||||
levelStr = "warn"
|
||||
|
||||
@@ -1,254 +0,0 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
type LoggingSuite struct {
|
||||
origLogger zerolog.Logger
|
||||
}
|
||||
|
||||
var _ = Suite(&LoggingSuite{})
|
||||
|
||||
func (s *LoggingSuite) SetUpTest(c *C) {
|
||||
// Save original logger
|
||||
s.origLogger = log.Logger
|
||||
}
|
||||
|
||||
func (s *LoggingSuite) TearDownTest(c *C) {
|
||||
// Restore original logger
|
||||
log.Logger = s.origLogger
|
||||
}
|
||||
|
||||
func (s *LoggingSuite) TestRunningOnTerminal(c *C) {
|
||||
// Test RunningOnTerminal function
|
||||
// The result depends on whether we're running in a terminal
|
||||
result := RunningOnTerminal()
|
||||
c.Check(result, FitsTypeOf, true)
|
||||
}
|
||||
|
||||
func (s *LoggingSuite) TestLogWriter(c *C) {
|
||||
// Test LogWriter struct and Write method
|
||||
var buf bytes.Buffer
|
||||
logger := zerolog.New(&buf)
|
||||
logWriter := LogWriter{Logger: logger}
|
||||
|
||||
// Test Write method
|
||||
testData := []byte("test log message")
|
||||
n, err := logWriter.Write(testData)
|
||||
c.Check(err, IsNil)
|
||||
c.Check(n, Equals, len(testData))
|
||||
|
||||
// Check that something was written to the buffer
|
||||
c.Check(buf.Len() > 0, Equals, true)
|
||||
|
||||
// Check that the output contains the message
|
||||
output := buf.String()
|
||||
c.Check(strings.Contains(output, "test log message"), Equals, true)
|
||||
}
|
||||
|
||||
func (s *LoggingSuite) TestLogWriterEmpty(c *C) {
|
||||
// Test LogWriter with empty data
|
||||
var buf bytes.Buffer
|
||||
logger := zerolog.New(&buf)
|
||||
logWriter := LogWriter{Logger: logger}
|
||||
|
||||
n, err := logWriter.Write([]byte{})
|
||||
c.Check(err, IsNil)
|
||||
c.Check(n, Equals, 0)
|
||||
}
|
||||
|
||||
func (s *LoggingSuite) TestSetupJSONLogger(c *C) {
|
||||
// Test SetupJSONLogger function
|
||||
var buf bytes.Buffer
|
||||
|
||||
// Test with different log levels
|
||||
testLevels := []string{"debug", "info", "warn", "error"}
|
||||
|
||||
for _, level := range testLevels {
|
||||
buf.Reset()
|
||||
SetupJSONLogger(level, &buf)
|
||||
|
||||
// Check that message and level field names are set correctly
|
||||
c.Check(zerolog.MessageFieldName, Equals, "message")
|
||||
c.Check(zerolog.LevelFieldName, Equals, "level")
|
||||
|
||||
// Test logging something
|
||||
log.Info().Msg("test message")
|
||||
|
||||
// Check that JSON was written
|
||||
output := buf.String()
|
||||
if len(output) > 0 {
|
||||
c.Check(strings.Contains(output, "message"), Equals, true)
|
||||
c.Check(strings.Contains(output, "test message"), Equals, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *LoggingSuite) TestSetupDefaultLogger(c *C) {
|
||||
// Test SetupDefaultLogger function
|
||||
testLevels := []string{"debug", "info", "warn", "error"}
|
||||
|
||||
for _, level := range testLevels {
|
||||
SetupDefaultLogger(level)
|
||||
|
||||
// Check that message and level field names are set correctly
|
||||
c.Check(zerolog.MessageFieldName, Equals, "message")
|
||||
c.Check(zerolog.LevelFieldName, Equals, "level")
|
||||
|
||||
// Check that logger is configured (hard to test output since it goes to stderr)
|
||||
c.Check(log.Logger, NotNil)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *LoggingSuite) TestGetLogLevelOrDebugValid(c *C) {
|
||||
// Test GetLogLevelOrDebug with valid levels
|
||||
testCases := map[string]zerolog.Level{
|
||||
"debug": zerolog.DebugLevel,
|
||||
"DEBUG": zerolog.DebugLevel,
|
||||
"info": zerolog.InfoLevel,
|
||||
"INFO": zerolog.InfoLevel,
|
||||
"warn": zerolog.WarnLevel,
|
||||
"WARN": zerolog.WarnLevel,
|
||||
"warning": zerolog.WarnLevel,
|
||||
"WARNING": zerolog.WarnLevel,
|
||||
"error": zerolog.ErrorLevel,
|
||||
"ERROR": zerolog.ErrorLevel,
|
||||
"fatal": zerolog.FatalLevel,
|
||||
"FATAL": zerolog.FatalLevel,
|
||||
"panic": zerolog.PanicLevel,
|
||||
"PANIC": zerolog.PanicLevel,
|
||||
"trace": zerolog.TraceLevel,
|
||||
"TRACE": zerolog.TraceLevel,
|
||||
}
|
||||
|
||||
for levelStr, expectedLevel := range testCases {
|
||||
result := GetLogLevelOrDebug(levelStr)
|
||||
c.Check(result, Equals, expectedLevel, Commentf("Failed for level: %s", levelStr))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *LoggingSuite) TestGetLogLevelOrDebugInvalid(c *C) {
|
||||
// Test GetLogLevelOrDebug with invalid levels
|
||||
invalidLevels := []string{
|
||||
"invalid",
|
||||
"unknown",
|
||||
"",
|
||||
"verbose",
|
||||
"critical",
|
||||
}
|
||||
|
||||
// Capture log output to verify warning is logged
|
||||
var buf bytes.Buffer
|
||||
originalLogger := log.Logger
|
||||
log.Logger = zerolog.New(&buf).Level(zerolog.TraceLevel)
|
||||
defer func() { log.Logger = originalLogger }()
|
||||
|
||||
for _, levelStr := range invalidLevels {
|
||||
buf.Reset()
|
||||
result := GetLogLevelOrDebug(levelStr)
|
||||
|
||||
// Should default to debug level
|
||||
c.Check(result, Equals, zerolog.DebugLevel, Commentf("Failed for invalid level: %s", levelStr))
|
||||
|
||||
// Should log a warning (if levelStr is not empty)
|
||||
if levelStr != "" {
|
||||
output := buf.String()
|
||||
c.Check(strings.Contains(output, "Unknown log level"), Equals, true, Commentf("No warning logged for: %s, got: %s", levelStr, output))
|
||||
c.Check(strings.Contains(output, levelStr), Equals, true, Commentf("Level not mentioned in warning: %s", levelStr))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *LoggingSuite) TestTimestampHook(c *C) {
|
||||
// Test timestampHook struct and Run method
|
||||
hook := ×tampHook{}
|
||||
|
||||
var buf bytes.Buffer
|
||||
logger := zerolog.New(&buf).Hook(hook)
|
||||
|
||||
// Log a message
|
||||
logger.Info().Msg("test message with timestamp")
|
||||
|
||||
// Check that output contains timestamp
|
||||
output := buf.String()
|
||||
c.Check(strings.Contains(output, "time"), Equals, true)
|
||||
c.Check(strings.Contains(output, "test message with timestamp"), Equals, true)
|
||||
|
||||
// Check that timestamp is in RFC3339 format (contains T and Z or +/- timezone)
|
||||
c.Check(strings.Contains(output, "T") || strings.Contains(output, ":"), Equals, true)
|
||||
}
|
||||
|
||||
func (s *LoggingSuite) TestTimestampHookMultipleLevels(c *C) {
|
||||
// Test timestampHook with different log levels
|
||||
hook := ×tampHook{}
|
||||
|
||||
var buf bytes.Buffer
|
||||
logger := zerolog.New(&buf).Hook(hook)
|
||||
|
||||
// Test different log levels
|
||||
testCases := []struct {
|
||||
level zerolog.Level
|
||||
message string
|
||||
}{
|
||||
{zerolog.DebugLevel, "debug message"},
|
||||
{zerolog.InfoLevel, "info message"},
|
||||
{zerolog.WarnLevel, "warn message"},
|
||||
{zerolog.ErrorLevel, "error message"},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
buf.Reset()
|
||||
logger.WithLevel(tc.level).Msg(tc.message)
|
||||
|
||||
output := buf.String()
|
||||
if len(output) > 0 {
|
||||
c.Check(strings.Contains(output, "time"), Equals, true, Commentf("No timestamp for level: %v", tc.level))
|
||||
c.Check(strings.Contains(output, tc.message), Equals, true, Commentf("Message missing for level: %v", tc.level))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *LoggingSuite) TestLogLevelCaseInsensitive(c *C) {
|
||||
// Test that log level parsing is case insensitive
|
||||
mixedCaseLevels := []string{
|
||||
"Debug", "deBuG", "DeBuG",
|
||||
"Info", "inFo", "InFo",
|
||||
"Warn", "waRn", "WaRn",
|
||||
"Error", "erRor", "ErRoR",
|
||||
}
|
||||
|
||||
for _, levelStr := range mixedCaseLevels {
|
||||
result := GetLogLevelOrDebug(levelStr)
|
||||
// Should not default to debug (unless it's actually debug)
|
||||
if strings.ToLower(levelStr) != "debug" {
|
||||
c.Check(result != zerolog.DebugLevel || strings.ToLower(levelStr) == "debug", Equals, true,
|
||||
Commentf("Case insensitive parsing failed for: %s", levelStr))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *LoggingSuite) TestSetupLoggersIntegration(c *C) {
|
||||
// Test integration between setup functions and actual logging
|
||||
var buf bytes.Buffer
|
||||
|
||||
// Test JSON logger
|
||||
SetupJSONLogger("info", &buf)
|
||||
log.Info().Str("key", "value").Msg("json test message")
|
||||
|
||||
jsonOutput := buf.String()
|
||||
if len(jsonOutput) > 0 {
|
||||
c.Check(strings.Contains(jsonOutput, "json test message"), Equals, true)
|
||||
c.Check(strings.Contains(jsonOutput, "key"), Equals, true)
|
||||
c.Check(strings.Contains(jsonOutput, "value"), Equals, true)
|
||||
}
|
||||
|
||||
// Test default logger (output goes to stderr, so we can't easily capture it)
|
||||
SetupDefaultLogger("warn")
|
||||
c.Check(log.Logger, NotNil)
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
type SanitizeSuite struct{}
|
||||
|
||||
var _ = Suite(&SanitizeSuite{})
|
||||
|
||||
func (s *SanitizeSuite) TestSanitizePath(c *C) {
|
||||
// Test various path scenarios based on actual implementation
|
||||
// The function removes "..", "$", "`" and leading "/"
|
||||
testCases := []struct {
|
||||
input string
|
||||
expected string
|
||||
desc string
|
||||
}{
|
||||
{
|
||||
input: "normal/path/file.txt",
|
||||
expected: "normal/path/file.txt",
|
||||
desc: "normal path unchanged",
|
||||
},
|
||||
{
|
||||
input: "../../../etc/passwd",
|
||||
expected: "etc/passwd",
|
||||
desc: ".. removed from path",
|
||||
},
|
||||
{
|
||||
input: "/absolute/path",
|
||||
expected: "absolute/path",
|
||||
desc: "leading slash removed",
|
||||
},
|
||||
{
|
||||
input: "path$with$dollar",
|
||||
expected: "pathwithdollar",
|
||||
desc: "dollar signs removed",
|
||||
},
|
||||
{
|
||||
input: "path`with`backticks",
|
||||
expected: "pathwithbackticks",
|
||||
desc: "backticks removed",
|
||||
},
|
||||
{
|
||||
input: "$HOME/.ssh/id_rsa",
|
||||
expected: "HOME/.ssh/id_rsa",
|
||||
desc: "environment variable syntax removed",
|
||||
},
|
||||
{
|
||||
input: "`echo malicious`",
|
||||
expected: "echo malicious",
|
||||
desc: "command substitution removed",
|
||||
},
|
||||
{
|
||||
input: "path/../other",
|
||||
expected: "path//other",
|
||||
desc: "internal .. removed leaving double slash",
|
||||
},
|
||||
{
|
||||
input: "",
|
||||
expected: "",
|
||||
desc: "empty path stays empty",
|
||||
},
|
||||
{
|
||||
input: "///multiple/leading/slashes",
|
||||
expected: "multiple/leading/slashes",
|
||||
desc: "multiple leading slashes removed",
|
||||
},
|
||||
{
|
||||
input: "test$../$../../../etc/passwd",
|
||||
expected: "test////etc/passwd",
|
||||
desc: "combined $ and .. removal",
|
||||
},
|
||||
{
|
||||
input: "/",
|
||||
expected: "",
|
||||
desc: "single slash becomes empty",
|
||||
},
|
||||
{
|
||||
input: "//",
|
||||
expected: "",
|
||||
desc: "double slash becomes empty",
|
||||
},
|
||||
{
|
||||
input: "valid/path/without/issues",
|
||||
expected: "valid/path/without/issues",
|
||||
desc: "valid path unchanged",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
result := SanitizePath(tc.input)
|
||||
c.Check(result, Equals, tc.expected, Commentf("Test case: %s", tc.desc))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SanitizeSuite) TestSanitizePathSecurity(c *C) {
|
||||
// Test specific security scenarios
|
||||
securityTests := []struct {
|
||||
input string
|
||||
desc string
|
||||
}{
|
||||
{"../../../../../../../../etc/shadow", "deep traversal"},
|
||||
{"../../../.ssh/id_rsa", "hidden file access"},
|
||||
{"./../../../root/.bashrc", "dotfile access"},
|
||||
{"$PATH/../../../../etc/hosts", "env var with traversal"},
|
||||
{"`id`/../../../etc/passwd", "command injection with traversal"},
|
||||
{"${HOME}/.aws/credentials", "env var syntax"},
|
||||
{"$(whoami)/../sensitive", "command substitution"},
|
||||
}
|
||||
|
||||
for _, tc := range securityTests {
|
||||
result := SanitizePath(tc.input)
|
||||
// After sanitization, should not contain dangerous patterns
|
||||
c.Check(strings.Contains(result, ".."), Equals, false, Commentf("Security test failed for: %s, got: %s", tc.desc, result))
|
||||
c.Check(strings.Contains(result, "$"), Equals, false, Commentf("Dollar sign present for: %s, got: %s", tc.desc, result))
|
||||
c.Check(strings.Contains(result, "`"), Equals, false, Commentf("Backtick present for: %s, got: %s", tc.desc, result))
|
||||
// Should not start with /
|
||||
if len(result) > 0 {
|
||||
c.Check(result[0] != '/', Equals, true, Commentf("Absolute path for: %s, got: %s", tc.desc, result))
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user