dbqcore

type BetweenRange struct {
	Min interface{}
	Max interface{}
}

type CheckExpression struct {
	FunctionName       string
	FunctionParameters []string
	Scope              CheckScope
	Operator           string
	ThresholdValue     interface{}
}

type CheckScope string

type ChecksFileConfig struct {
	Version string           `yaml:"version"`
	Rules   []ValidationRule `yaml:"rules"`
}

type ColumnInfo struct {
	Name     string
	Type     string
	Comment  string
	Position uint
}

type ColumnMetrics struct {
	ColumnName          string   `json:"col_name"`
	ColumnComment       string   `json:"col_comment"`
	ColumnPosition      uint     `json:"col_position"`
	DataType            string   `json:"data_type"`
	NullCount           uint64   `json:"null_count"`
	BlankCount          *int64   `json:"blank_count,omitempty"`         // string only
	MinValue            *float64 `json:"min_value,omitempty"`           // numeric only
	MaxValue            *float64 `json:"max_value,omitempty"`           // numeric only
	AvgValue            *float64 `json:"avg_value,omitempty"`           // numeric only
	StddevValue         *float64 `json:"stddev_value,omitempty"`        // numeric only (Population StdDev)
	MostFrequentValue   *string  `json:"most_frequent_value,omitempty"` // pointer to handle NULL as most frequent
	ProfilingDurationMs int64    `json:"profiling_duration_ms"`
}

type ColumnsNotPresentConfig struct {
	Columns []string `yaml:"columns,omitempty"`
	Pattern string   `yaml:"pattern,omitempty"`
}

type ConnectionConfig struct {
	Host     string `yaml:"host"`
	Port     int    `yaml:"port"`
	Username string `yaml:"username"`
	Password string `yaml:"password"`
	Database string `yaml:"database,omitempty"`
}

type DataQualityCheck struct {
	Expression  string       `yaml:"-"`
	Description string       `yaml:"desc,omitempty"`
	OnFail      OnFailAction `yaml:"on_fail,omitempty"`
	Query       string       `yaml:"query,omitempty"`

	// Schema check fields
	SchemaCheck *SchemaCheckConfig `yaml:"schema_check,omitempty"`
	ParsedCheck *CheckExpression   `yaml:"-"`
}

type DataQualityCheckType string

type DataSource struct {
	ID            string           `yaml:"id"`
	Type          DataSourceType   `yaml:"type"`
	Configuration ConnectionConfig `yaml:"configuration"`
	Datasets      []string         `yaml:"datasets"`
}

type DataSourceType string

type DbqConfig struct {
	Version     string       `yaml:"version"`
	DataSources []DataSource `yaml:"datasources"`
}

type DbqConnector interface {
	// Ping checks if the connection to the data source is alive.
	Ping(ctx context.Context) (string, error)

	// ImportDatasets imports datasets from the data source, with an optional filter.
	ImportDatasets(ctx context.Context, filter string) ([]string, error)
}

type DbqDataProfiler interface {
	// ProfileDataset is an entry point that runs profiling process by tying all specific profiling calls together
	// todo: consider extracting it into separate entity
	ProfileDataset(ctx context.Context, dataset string, sample bool, maxConcurrent int, collectErrors bool) (*TableMetrics, error)

	GetColumns(ctx context.Context, databaseName string, tableName string) ([]*ColumnInfo, error)
	GetTotalRows(ctx context.Context, dataset string) (uint64, error)
	GetNullCount(ctx context.Context, dataset string, column *ColumnInfo) (uint64, error)
	GetBlankCount(ctx context.Context, dataset string, column *ColumnInfo) (int64, error)
	GetNumericStats(ctx context.Context, dataset string, column *ColumnInfo) (*NumericStats, error)
	GetMostFrequentValue(ctx context.Context, dataset string, column *ColumnInfo) (*string, error)
	GetSampleData(ctx context.Context, dataset string) ([]map[string]interface{}, error)
	IsNumericType(dataType string) bool
	IsStringType(dataType string) bool
}

type DbqDataSourceAdapter interface {
	// InterpretDataQualityCheck generates a SQL query specific for datasource for a data quality check
	InterpretDataQualityCheck(check *DataQualityCheck, dataset string, defaultWhere string) (string, error)

	// ExecuteQuery executes the SQL query and returns the query result
	ExecuteQuery(ctx context.Context, query string) (interface{}, error)
}

type DbqDataValidator interface {
	// RunCheck runs a data quality check and returns the result.
	RunCheck(ctx context.Context, adapter DbqDataSourceAdapter, check *DataQualityCheck, dataset string, defaultWhere string) *ValidationResult
}

type DbqDataValidatorImpl struct {
	// contains filtered or unexported fields
}

type ExpectColumnsConfig struct {
	Columns []string `yaml:"columns"`
}

type ExpectColumnsOrderedConfig struct {
	ColumnsOrder []string `yaml:"columns_order"`
}

type NumericStats struct {
	MinValue    *float64
	MaxValue    *float64
	AvgValue    *float64
	StddevValue *float64
}

type OnFailAction string

type SchemaCheckConfig struct {
	ExpectColumnsOrdered *ExpectColumnsOrderedConfig `yaml:"expect_columns_ordered,omitempty"`
	ExpectColumns        *ExpectColumnsConfig        `yaml:"expect_columns,omitempty"`
	ColumnsNotPresent    *ColumnsNotPresentConfig    `yaml:"columns_not_present,omitempty"`
}

type TableMetrics struct {
	ProfiledAt          int64                     `json:"profiled_at"`
	TableName           string                    `json:"table_name"`
	DatabaseName        string                    `json:"database_name"`
	TotalRows           uint64                    `json:"total_rows"`
	ColumnsMetrics      map[string]*ColumnMetrics `json:"columns_metrics"`
	RowsSample          []map[string]interface{}  `json:"rows_sample"`
	ProfilingDurationMs int64                     `json:"profiling_duration_ms"`
	DbqErrors           []error                   `json:"__dbq_errors"`
}

type TaskPool struct {
	// contains filtered or unexported fields
}

type ValidationResult struct {
	CheckID          string `json:"check_id"`
	Pass             bool   `json:"pass"`
	QueryResultValue string `json:"query_result_value,omitempty"`
	Error            string `json:"error,omitempty"`
}

type ValidationRule struct {
	Dataset string             `yaml:"dataset"`
	Where   string             `yaml:"where,omitempty"`
	Checks  []DataQualityCheck `yaml:"checks"`
}