userconfig

package
v0.2.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 12, 2019 License: Apache-2.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	UnknownKey         = "unknown"
	NameKey            = "name"
	KindKey            = "kind"
	DataKey            = "data"
	SchemaKey          = "schema"
	ColumnsKey         = "columns"
	FeatureColumnsKey  = "feature_columns"
	TrainingColumnsKey = "training_columns"
	TargetColumnKey    = "target_column"
	AggregatesKey      = "aggregates"
	ModelNameKey       = "model_name"
	InputsKey          = "inputs"
	ArgsKey            = "args"
	TypeKey            = "type"
	AggregatorKey      = "aggregator"
	TransformerKey     = "transformer"
	PathKey            = "path"
	ValueKey           = "value"
	YAMLKey            = "yaml"

	// environment
	LimitKey          = "limit"
	NumRowsKey        = "num_rows"
	FractionOfRowsKey = "fraction_of_rows"
	RandomizeKey      = "randomize"
	RandomSeedKey     = "random_seed"

	// model
	NumEpochsKey           = "num_epochs"
	NumStepsKey            = "num_steps"
	SaveCheckpointSecsKey  = "save_checkpoints_secs"
	SaveCheckpointStepsKey = "save_checkpoints_steps"
	DataPartitionRatioKey  = "data_partition_ratio"
	TrainingKey            = "training"
	EvaluationKey          = "evaluation"
)

Variables

This section is empty.

Functions

func CastValue 

func CastValue(value interface{}, valueType interface{}) (interface{}, error)

func CheckArgRuntimeTypesMatch 

func CheckArgRuntimeTypesMatch(argRuntimeTypes map[string]interface{}, argSchemaTypes map[string]interface{}) error

func CheckColumnRuntimeTypesMatch 

func CheckColumnRuntimeTypesMatch(columnRuntimeTypes map[string]interface{}, columnSchemaTypes map[string]interface{}) error

columnRuntimeTypes is {string -> ColumnType or []ColumnType}, columnSchemaTypes is {string -> string or []string}

func CheckValueRuntimeTypesMatch 

func CheckValueRuntimeTypesMatch(runtimeType interface{}, schemaType interface{}) error

func ColumnTypeStrings 

func ColumnTypeStrings() []string

func EnvironmentDataTypeStrings added in v0.2.0 

func EnvironmentDataTypeStrings() []string

func ErrorArgNameCannotBeType 

func ErrorArgNameCannotBeType(provided string) error

func ErrorColumnMustBeRaw 

func ErrorColumnMustBeRaw(columnName string) error

func ErrorDuplicateConfig 

func ErrorDuplicateConfig(resourceType resource.Type) error

func ErrorDuplicateResourceName added in v0.2.0 

func ErrorDuplicateResourceName(resources ...Resource) error

func ErrorDuplicateResourceValue 

func ErrorDuplicateResourceValue(value string, keys ...string) error

func ErrorGenericTypeMapLength 

func ErrorGenericTypeMapLength(provided interface{}) error

func ErrorInvalidColumnInputType 

func ErrorInvalidColumnInputType(provided interface{}) error

func ErrorInvalidColumnRuntimeType 

func ErrorInvalidColumnRuntimeType() error

func ErrorInvalidValueDataType 

func ErrorInvalidValueDataType(provided interface{}) error

func ErrorK8sQuantityMustBeInt 

func ErrorK8sQuantityMustBeInt(quantityStr string) error

func ErrorMalformedConfig 

func ErrorMalformedConfig() error

func ErrorMissingAppDefinition 

func ErrorMissingAppDefinition() error

func ErrorMissingRawColumns 

func ErrorMissingRawColumns(missingColumns []string) error

func ErrorOneOfPrerequisitesNotDefined added in v0.2.0 

func ErrorOneOfPrerequisitesNotDefined(argName string, prerequisites ...string) error

func ErrorParseConfig 

func ErrorParseConfig() error

func ErrorReadConfig 

func ErrorReadConfig() error

func ErrorSpecifyAllOrNone 

func ErrorSpecifyAllOrNone(vals ...string) error

func ErrorSpecifyOnlyOne 

func ErrorSpecifyOnlyOne(vals ...string) error

func ErrorTemplateExtraArg 

func ErrorTemplateExtraArg(template *Template, argName string) error

func ErrorTemplateMissingArg 

func ErrorTemplateMissingArg(template *Template, argName string) error

func ErrorTypeListLength 

func ErrorTypeListLength(provided interface{}) error

func ErrorUndefinedConfig 

func ErrorUndefinedConfig(resourceType resource.Type) error

func ErrorUndefinedResource 

func ErrorUndefinedResource(resourceName string, resourceTypes ...resource.Type) error

func ErrorUndefinedResourceBuiltin 

func ErrorUndefinedResourceBuiltin(resourceName string, resourceTypes ...resource.Type) error

func ErrorUnsupportedColumnType 

func ErrorUnsupportedColumnType(provided interface{}, allowedTypes []string) error

func ErrorUnsupportedDataType 

func ErrorUnsupportedDataType(provided interface{}, allowedType interface{}) error

func Identify 

func Identify(r Resource) string

func ModelTypeStrings added in v0.2.0 

func ModelTypeStrings() []string

func QuantityParser 

func QuantityParser(v *QuantityValidation) func(string) (interface{}, error)

func QuantityPtrID 

func QuantityPtrID(quantity *Quantity) string

func QuantityPtrsEqual 

func QuantityPtrsEqual(quantity *Quantity, quantity2 *Quantity) bool

func ReadAppName 

func ReadAppName(filePath string, relativePath string) (string, error)

func ValidateArgTypes 

func ValidateArgTypes(argTypes map[string]interface{}) error

func ValidateArgValues 

func ValidateArgValues(argValues map[string]interface{}) error

func ValidateColumnInputTypes 

func ValidateColumnInputTypes(columnTypes map[string]interface{}) error

func ValidateColumnInputValues 

func ValidateColumnInputValues(columnInputValues map[string]interface{}) error

func ValidateColumnInputsExistAndRaw 

func ValidateColumnInputsExistAndRaw(columnInputValues map[string]interface{}, config *Config) error

func ValidateColumnNameExistsAndRaw 

func ValidateColumnNameExistsAndRaw(columnName string, config *Config) error

func ValidateColumnRuntimeTypes 

func ValidateColumnRuntimeTypes(columnRuntimeTypes map[string]interface{}) error

func ValidateValue 

func ValidateValue(value interface{}) error

func ValidateValueType 

func ValidateValueType(valueType interface{}) error

func ValueTypeStrings 

func ValueTypeStrings() []string

Types

type API 

type API struct {
	ResourceConfigFields
	ModelName string      `json:"model_name" yaml:"model_name"`
	Compute   *APICompute `json:"compute" yaml:"compute"`
	Tags      Tags        `json:"tags" yaml:"tags"`
}

func (*API) GetResourceType 

func (api *API) GetResourceType() resource.Type

type APICompute 

type APICompute struct {
	Replicas int32     `json:"replicas" yaml:"replicas"`
	CPU      *Quantity `json:"cpu" yaml:"cpu"`
	Mem      *Quantity `json:"mem" yaml:"mem"`
	GPU      int64     `json:"gpu" yaml:"gpu"`
}

func (*APICompute) Equal 

func (apiCompute *APICompute) Equal(apiCompute2 APICompute) bool

func (*APICompute) ID 

func (apiCompute *APICompute) ID() string

func (*APICompute) IDWithoutReplicas 

func (apiCompute *APICompute) IDWithoutReplicas() string

type APIs 

type APIs []*API

func (APIs) Names 

func (apis APIs) Names() []string

func (APIs) Validate 

func (apis APIs) Validate() error

type Aggregate 

type Aggregate struct {
	ResourceConfigFields
	Aggregator string        `json:"aggregator" yaml:"aggregator"`
	Inputs     *Inputs       `json:"inputs" yaml:"inputs"`
	Compute    *SparkCompute `json:"compute" yaml:"compute"`
	Tags       Tags          `json:"tags" yaml:"tags"`
}

func (*Aggregate) GetResourceType 

func (aggregate *Aggregate) GetResourceType() resource.Type

func (*Aggregate) InputColumnNames 

func (aggregate *Aggregate) InputColumnNames() []string

type Aggregates 

type Aggregates []*Aggregate

func (Aggregates) Get 

func (aggregates Aggregates) Get(name string) *Aggregate

func (Aggregates) Names 

func (aggregates Aggregates) Names() []string

func (Aggregates) Validate 

func (aggregates Aggregates) Validate() error

type Aggregator 

type Aggregator struct {
	ResourceConfigFields
	Inputs     *Inputs     `json:"inputs"  yaml:"inputs"`
	OutputType interface{} `json:"output_type"  yaml:"output_type"`
	Path       string      `json:"path"  yaml:"path"`
}

func (*Aggregator) GetResourceType 

func (aggregator *Aggregator) GetResourceType() resource.Type

type Aggregators 

type Aggregators []*Aggregator

func (Aggregators) Get 

func (aggregators Aggregators) Get(name string) *Aggregator

func (Aggregators) Names 

func (aggregators Aggregators) Names() []string

func (Aggregators) Validate 

func (aggregators Aggregators) Validate() error

type App 

type App struct {
	Name string `json:"name" yaml:"name"`
}

func (*App) Validate 

func (app *App) Validate() error

type CSVConfig added in v0.2.0 

type CSVConfig struct {
	Sep                       *string `json:"sep" yaml:"sep"`
	Encoding                  *string `json:"encoding" yaml:"encoding"`
	Quote                     *string `json:"quote" yaml:"quote"`
	Escape                    *string `json:"escape" yaml:"escape"`
	Comment                   *string `json:"comment" yaml:"comment"`
	Header                    *bool   `json:"header" yaml:"header"`
	IgnoreLeadingWhiteSpace   *bool   `json:"ignore_leading_white_space" yaml:"ignore_leading_white_space"`
	IgnoreTrailingWhiteSpace  *bool   `json:"ignore_trailing_white_space" yaml:"ignore_trailing_white_space"`
	NullValue                 *string `json:"null_value" yaml:"null_value"`
	NanValue                  *string `json:"nan_value" yaml:"nan_value"`
	PositiveInf               *string `json:"positive_inf" yaml:"positive_inf"`
	NegativeInf               *string `json:"negative_inf" yaml:"negative_inf"`
	MaxColumns                *int32  `json:"max_columns" yaml:"max_columns"`
	MaxCharsPerColumn         *int32  `json:"max_chars_per_column" yaml:"max_chars_per_column"`
	Multiline                 *bool   `json:"multiline" yaml:"multiline"`
	CharToEscapeQuoteEscaping *string `json:"char_to_escape_quote_escaping" yaml:"char_to_escape_quote_escaping"`
	EmptyValue                *string `json:"empty_value" yaml:"empty_value"`
}

CSVConfig is SPARK_VERSION dependent

type CSVData added in v0.2.0 

type CSVData struct {
	Type      EnvironmentDataType `json:"type" yaml:"type"`
	Path      string              `json:"path" yaml:"path"`
	Schema    []string            `json:"schema" yaml:"schema"`
	DropNull  bool                `json:"drop_null" yaml:"drop_null"`
	CSVConfig *CSVConfig          `json:"csv_config" yaml:"csv_config"`
}

func (*CSVData) GetExternalPath added in v0.2.0 

func (csvData *CSVData) GetExternalPath() string

func (*CSVData) GetIngestedColumns added in v0.2.0 

func (csvData *CSVData) GetIngestedColumns() []string

func (*CSVData) Validate added in v0.2.0 

func (csvData *CSVData) Validate() error

type Column 

type Column interface {
	Resource
	IsRaw() bool
}

type ColumnType 

type ColumnType int
const (
	UnknownColumnType ColumnType = iota
	IntegerColumnType
	FloatColumnType
	StringColumnType
	IntegerListColumnType
	FloatListColumnType
	StringListColumnType
)

func ColumnTypeFromString 

func ColumnTypeFromString(s string) ColumnType

func (ColumnType) JSONPlaceholder added in v0.2.0 

func (t ColumnType) JSONPlaceholder() string

func (ColumnType) MarshalBinary 

func (t ColumnType) MarshalBinary() ([]byte, error)

MarshalBinary satisfies BinaryMarshaler

func (ColumnType) MarshalText 

func (t ColumnType) MarshalText() ([]byte, error)

MarshalText satisfies TextMarshaler

func (ColumnType) String 

func (t ColumnType) String() string

func (*ColumnType) UnmarshalBinary 

func (t *ColumnType) UnmarshalBinary(data []byte) error

UnmarshalBinary satisfies BinaryUnmarshaler Needed for msgpack

func (*ColumnType) UnmarshalText 

func (t *ColumnType) UnmarshalText(text []byte) error

UnmarshalText satisfies TextUnmarshaler

type ColumnTypes 

type ColumnTypes []ColumnType

func (ColumnTypes) String 

func (ts ColumnTypes) String() string

func (ColumnTypes) StringList 

func (ts ColumnTypes) StringList() []string

type Config 

type Config struct {
	App                *App               `json:"app" yaml:"app"`
	Environments       Environments       `json:"environments" yaml:"environments"`
	Environment        *Environment       `json:"environment" yaml:"environment"`
	RawColumns         RawColumns         `json:"raw_columns" yaml:"raw_columns"`
	Aggregates         Aggregates         `json:"aggregates" yaml:"aggregates"`
	TransformedColumns TransformedColumns `json:"transformed_columns" yaml:"transformed_columns"`
	Models             Models             `json:"models" yaml:"models"`
	APIs               APIs               `json:"apis" yaml:"apis"`
	Aggregators        Aggregators        `json:"aggregators" yaml:"aggregators"`
	Transformers       Transformers       `json:"transformers" yaml:"transformers"`
	Constants          Constants          `json:"constants" yaml:"constants"`
	Templates          Templates          `json:"templates" yaml:"templates"`
	Embeds             Embeds             `json:"embeds" yaml:"embeds"`
}

func New 

func New(configs map[string][]byte, envName string) (*Config, error)

func NewPartialPath 

func NewPartialPath(filePath string) (*Config, error)

func (*Config) ColumnNames 

func (config *Config) ColumnNames() []string

func (*Config) IsRawColumn 

func (config *Config) IsRawColumn(name string) bool

func (*Config) IsTransformedColumn 

func (config *Config) IsTransformedColumn(name string) bool

func (*Config) MergeBytes 

func (config *Config) MergeBytes(configBytes []byte, filePath string, emb *Embed, template *Template) (*Config, error)

func (*Config) Validate 

func (config *Config) Validate(envName string) error

func (*Config) ValidateColumns 

func (config *Config) ValidateColumns() error

func (*Config) ValidatePartial 

func (config *Config) ValidatePartial() error

type Constant 

type Constant struct {
	ResourceConfigFields
	Type  interface{} `json:"type" yaml:"type"`
	Value interface{} `json:"value" yaml:"value"`
	Tags  Tags        `json:"tags" yaml:"tags"`
}

func (*Constant) GetResourceType 

func (constant *Constant) GetResourceType() resource.Type

func (*Constant) GetType 

func (constant *Constant) GetType() interface{}

func (*Constant) Validate 

func (constant *Constant) Validate() error

type Constants 

type Constants []*Constant

func (Constants) Names 

func (constants Constants) Names() []string

func (Constants) Validate 

func (constants Constants) Validate() error

type Data 

type Data interface {
	GetIngestedColumns() []string
	GetExternalPath() string
	Validate() error
}

type Embed 

type Embed struct {
	ResourceConfigFields
	Template string                 `json:"template" yaml:"template"`
	Args     map[string]interface{} `json:"args" yaml:"args"`
}

func (*Embed) GetResourceType added in v0.2.0 

func (embed *Embed) GetResourceType() resource.Type

type Embeds 

type Embeds []*Embed

type Environment 

type Environment struct {
	ResourceConfigFields
	LogLevel *LogLevel `json:"log_level" yaml:"log_level"`
	Limit    *Limit    `json:"limit" yaml:"limit"`
	Data     Data      `json:"-" yaml:"-"`
}

func (*Environment) GetResourceType 

func (env *Environment) GetResourceType() resource.Type

func (*Environment) Validate 

func (env *Environment) Validate() error

type EnvironmentDataType added in v0.2.0 

type EnvironmentDataType int
const (
	UnknownEnvironmentDataType EnvironmentDataType = iota
	CSVEnvironmentDataType
	ParquetEnvironmentDataType
)

func EnvironmentDataTypeFromString added in v0.2.0 

func EnvironmentDataTypeFromString(s string) EnvironmentDataType

func (EnvironmentDataType) MarshalBinary added in v0.2.0 

func (t EnvironmentDataType) MarshalBinary() ([]byte, error)

MarshalBinary satisfies BinaryMarshaler

func (EnvironmentDataType) MarshalText added in v0.2.0 

func (t EnvironmentDataType) MarshalText() ([]byte, error)

MarshalText satisfies TextMarshaler

func (EnvironmentDataType) String added in v0.2.0 

func (t EnvironmentDataType) String() string

func (*EnvironmentDataType) UnmarshalBinary added in v0.2.0 

func (t *EnvironmentDataType) UnmarshalBinary(data []byte) error

UnmarshalBinary satisfies BinaryUnmarshaler Needed for msgpack

func (*EnvironmentDataType) UnmarshalText added in v0.2.0 

func (t *EnvironmentDataType) UnmarshalText(text []byte) error

UnmarshalText satisfies TextUnmarshaler

type Environments 

type Environments []*Environment

func (Environments) Names 

func (environments Environments) Names() []string

func (Environments) Validate 

func (environments Environments) Validate() error

type Error added in v0.2.0 

type Error struct {
	Kind ErrorKind
	// contains filtered or unexported fields
}

func (Error) Error added in v0.2.0 

func (e Error) Error() string

type ErrorKind 

type ErrorKind int
const (
	ErrUnknown ErrorKind = iota
	ErrDuplicateConfigName
	ErrDuplicateResourceValue
	ErrDuplicateConfig
	ErrMalformedConfig
	ErrParseConfig
	ErrReadConfig
	ErrMissingAppDefinition
	ErrUndefinedConfig
	ErrMissingRawColumns
	ErrUndefinedResource
	ErrUndefinedResourceBuiltin
	ErrColumnMustBeRaw
	ErrSpecifyAllOrNone
	ErrSpecifyOnlyOne
	ErrOneOfPrerequisitesNotDefined
	ErrTemplateExtraArg
	ErrTemplateMissingArg
	ErrInvalidColumnInputType
	ErrInvalidColumnRuntimeType
	ErrInvalidValueDataType
	ErrUnsupportedColumnType
	ErrUnsupportedDataType
	ErrArgNameCannotBeType
	ErrTypeListLength
	ErrGenericTypeMapLength
	ErrK8sQuantityMustBeInt
)

func (ErrorKind) MarshalBinary 

func (t ErrorKind) MarshalBinary() ([]byte, error)

MarshalBinary satisfies BinaryMarshaler

func (ErrorKind) MarshalText 

func (t ErrorKind) MarshalText() ([]byte, error)

MarshalText satisfies TextMarshaler

func (ErrorKind) String 

func (t ErrorKind) String() string

func (*ErrorKind) UnmarshalBinary 

func (t *ErrorKind) UnmarshalBinary(data []byte) error

UnmarshalBinary satisfies BinaryUnmarshaler Needed for msgpack

func (*ErrorKind) UnmarshalText 

func (t *ErrorKind) UnmarshalText(text []byte) error

UnmarshalText satisfies TextUnmarshaler

type Inputs 

type Inputs struct {
	Columns map[string]interface{} `json:"columns" yaml:"columns"`
	Args    map[string]interface{} `json:"args" yaml:"args"`
}

type Limit added in v0.2.0 

type Limit struct {
	NumRows        *int64   `json:"num_rows" yaml:"num_rows"`
	FractionOfRows *float32 `json:"fraction_of_rows" yaml:"fraction_of_rows"`
	Randomize      *bool    `json:"randomize" yaml:"randomize"`
	RandomSeed     *int64   `json:"random_seed" yaml:"random_seed"`
}

type LogLevel 

type LogLevel struct {
	Tensorflow string `json:"tensorflow" yaml:"tensorflow"`
	Spark      string `json:"spark" yaml:"spark"`
}

type Model 

type Model struct {
	ResourceConfigFields
	Type               ModelType                `json:"type" yaml:"type"`
	Path               string                   `json:"path" yaml:"path"`
	TargetColumn       string                   `json:"target_column" yaml:"target_column"`
	PredictionKey      string                   `json:"prediction_key" yaml:"prediction_key"`
	FeatureColumns     []string                 `json:"feature_columns" yaml:"feature_columns"`
	TrainingColumns    []string                 `json:"training_columns" yaml:"training_columns"`
	Aggregates         []string                 `json:"aggregates"  yaml:"aggregates"`
	Hparams            map[string]interface{}   `json:"hparams" yaml:"hparams"`
	DataPartitionRatio *ModelDataPartitionRatio `json:"data_partition_ratio" yaml:"data_partition_ratio"`
	Training           *ModelTraining           `json:"training" yaml:"training"`
	Evaluation         *ModelEvaluation         `json:"evaluation" yaml:"evaluation"`
	Compute            *TFCompute               `json:"compute" yaml:"compute"`
	Tags               Tags                     `json:"tags" yaml:"tags"`
}

func (*Model) AllColumnNames 

func (model *Model) AllColumnNames() []string

func (*Model) GetResourceType 

func (model *Model) GetResourceType() resource.Type

func (*Model) Validate 

func (model *Model) Validate() error

type ModelDataPartitionRatio 

type ModelDataPartitionRatio struct {
	Training   *float64 `json:"training"`
	Evaluation *float64 `json:"evaluation"`
}

type ModelEvaluation 

type ModelEvaluation struct {
	BatchSize      int64  `json:"batch_size" yaml:"batch_size"`
	NumSteps       *int64 `json:"num_steps" yaml:"num_steps"`
	NumEpochs      *int64 `json:"num_epochs" yaml:"num_epochs"`
	Shuffle        bool   `json:"shuffle" yaml:"shuffle"`
	StartDelaySecs int64  `json:"start_delay_secs" yaml:"start_delay_secs"`
	ThrottleSecs   int64  `json:"throttle_secs" yaml:"throttle_secs"`
}

type ModelTraining 

type ModelTraining struct {
	BatchSize                 int64  `json:"batch_size" yaml:"batch_size"`
	NumSteps                  *int64 `json:"num_steps" yaml:"num_steps"`
	NumEpochs                 *int64 `json:"num_epochs" yaml:"num_epochs"`
	Shuffle                   bool   `json:"shuffle" yaml:"shuffle"`
	TfRandomSeed              int64  `json:"tf_random_seed" yaml:"tf_random_seed"`
	TfRandomizeSeed           bool   `json:"tf_randomize_seed" yaml:"tf_randomize_seed"`
	SaveSummarySteps          int64  `json:"save_summary_steps" yaml:"save_summary_steps"`
	SaveCheckpointsSecs       *int64 `json:"save_checkpoints_secs" yaml:"save_checkpoints_secs"`
	SaveCheckpointsSteps      *int64 `json:"save_checkpoints_steps" yaml:"save_checkpoints_steps"`
	LogStepCountSteps         int64  `json:"log_step_count_steps" yaml:"log_step_count_steps"`
	KeepCheckpointMax         int64  `json:"keep_checkpoint_max" yaml:"keep_checkpoint_max"`
	KeepCheckpointEveryNHours int64  `json:"keep_checkpoint_every_n_hours" yaml:"keep_checkpoint_every_n_hours"`
}

type ModelType added in v0.2.0 

type ModelType int
const (
	UnknownModelType ModelType = iota
	ClassificationModelType
	RegressionModelType
)

func ModelTypeFromString added in v0.2.0 

func ModelTypeFromString(s string) ModelType

func (ModelType) MarshalBinary added in v0.2.0 

func (t ModelType) MarshalBinary() ([]byte, error)

MarshalBinary satisfies BinaryMarshaler

func (ModelType) MarshalText added in v0.2.0 

func (t ModelType) MarshalText() ([]byte, error)

MarshalText satisfies TextMarshaler

func (ModelType) String added in v0.2.0 

func (t ModelType) String() string

func (*ModelType) UnmarshalBinary added in v0.2.0 

func (t *ModelType) UnmarshalBinary(data []byte) error

UnmarshalBinary satisfies BinaryUnmarshaler Needed for msgpack

func (*ModelType) UnmarshalText added in v0.2.0 

func (t *ModelType) UnmarshalText(text []byte) error

UnmarshalText satisfies TextUnmarshaler

type Models 

type Models []*Model

func (Models) Names 

func (models Models) Names() []string

func (Models) Validate 

func (models Models) Validate() error

type ParquetColumn 

type ParquetColumn struct {
	ParquetColumnName string `json:"parquet_column_name" yaml:"parquet_column_name"`
	RawColumnName     string `json:"raw_column_name" yaml:"raw_column_name"`
}

type ParquetData 

type ParquetData struct {
	Type     EnvironmentDataType `json:"type" yaml:"type"`
	Path     string              `json:"path" yaml:"path"`
	Schema   []*ParquetColumn    `json:"schema" yaml:"schema"`
	DropNull bool                `json:"drop_null" yaml:"drop_null"`
}

func (*ParquetData) GetExternalPath 

func (parqData *ParquetData) GetExternalPath() string

func (*ParquetData) GetIngestedColumns 

func (parqData *ParquetData) GetIngestedColumns() []string

func (*ParquetData) Validate 

func (parqData *ParquetData) Validate() error

type Quantity 

type Quantity struct {
	k8sresource.Quantity
	UserString string
}

func (*Quantity) Equal 

func (quantity *Quantity) Equal(quantity2 Quantity) bool

func (*Quantity) ID 

func (quantity *Quantity) ID() string

func (Quantity) MarshalBinary 

func (quantity Quantity) MarshalBinary() ([]byte, error)

func (Quantity) MarshalJSON 

func (quantity Quantity) MarshalJSON() ([]byte, error)

func (Quantity) MarshalText 

func (quantity Quantity) MarshalText() ([]byte, error)

func (*Quantity) SplitInTwo 

func (quantity *Quantity) SplitInTwo() (*k8sresource.Quantity, *k8sresource.Quantity)

SplitInTwo divides the quantity in two and return both halves (ensuring they add up to the original value)

func (*Quantity) String 

func (quantity *Quantity) String() string

func (*Quantity) ToFloat32 

func (quantity *Quantity) ToFloat32() float32

func (*Quantity) ToKi 

func (quantity *Quantity) ToKi() int64

func (*Quantity) UnmarshalBinary 

func (quantity *Quantity) UnmarshalBinary(data []byte) error

func (*Quantity) UnmarshalJSON 

func (quantity *Quantity) UnmarshalJSON(data []byte) error

func (*Quantity) UnmarshalText 

func (quantity *Quantity) UnmarshalText(data []byte) error

type QuantityValidation 

type QuantityValidation struct {
	Min k8sresource.Quantity
	Int bool
}

type RawColumn 

type RawColumn interface {
	Column
	GetType() ColumnType
	GetCompute() *SparkCompute
	GetUserConfig() Resource
}

type RawColumns 

type RawColumns []RawColumn

func (RawColumns) Get 

func (rawColumns RawColumns) Get(name string) RawColumn

func (RawColumns) Names 

func (rawColumns RawColumns) Names() []string

func (RawColumns) Validate 

func (rawColumns RawColumns) Validate() error

type RawFloatColumn 

type RawFloatColumn struct {
	ResourceConfigFields
	Type     ColumnType    `json:"type" yaml:"type"`
	Required bool          `json:"required" yaml:"required"`
	Min      *float32      `json:"min" yaml:"min"`
	Max      *float32      `json:"max" yaml:"max"`
	Values   []float32     `json:"values" yaml:"values"`
	Compute  *SparkCompute `json:"compute" yaml:"compute"`
	Tags     Tags          `json:"tags" yaml:"tags"`
}

func (*RawFloatColumn) GetCompute 

func (column *RawFloatColumn) GetCompute() *SparkCompute

func (*RawFloatColumn) GetResourceType 

func (column *RawFloatColumn) GetResourceType() resource.Type

func (*RawFloatColumn) GetType 

func (column *RawFloatColumn) GetType() ColumnType

func (*RawFloatColumn) GetUserConfig 

func (column *RawFloatColumn) GetUserConfig() Resource

func (*RawFloatColumn) IsRaw 

func (column *RawFloatColumn) IsRaw() bool

type RawIntColumn 

type RawIntColumn struct {
	ResourceConfigFields
	Type     ColumnType    `json:"type" yaml:"type"`
	Required bool          `json:"required" yaml:"required"`
	Min      *int64        `json:"min" yaml:"min"`
	Max      *int64        `json:"max" yaml:"max"`
	Values   []int64       `json:"values" yaml:"values"`
	Compute  *SparkCompute `json:"compute" yaml:"compute"`
	Tags     Tags          `json:"tags" yaml:"tags"`
}

func (*RawIntColumn) GetCompute 

func (column *RawIntColumn) GetCompute() *SparkCompute

func (*RawIntColumn) GetResourceType 

func (column *RawIntColumn) GetResourceType() resource.Type

func (*RawIntColumn) GetType 

func (column *RawIntColumn) GetType() ColumnType

func (*RawIntColumn) GetUserConfig 

func (column *RawIntColumn) GetUserConfig() Resource

func (*RawIntColumn) IsRaw 

func (column *RawIntColumn) IsRaw() bool

type RawStringColumn 

type RawStringColumn struct {
	ResourceConfigFields
	Type     ColumnType    `json:"type" yaml:"type"`
	Required bool          `json:"required" yaml:"required"`
	Values   []string      `json:"values" yaml:"values"`
	Compute  *SparkCompute `json:"compute" yaml:"compute"`
	Tags     Tags          `json:"tags" yaml:"tags"`
}

func (*RawStringColumn) GetCompute 

func (column *RawStringColumn) GetCompute() *SparkCompute

func (*RawStringColumn) GetResourceType 

func (column *RawStringColumn) GetResourceType() resource.Type

func (*RawStringColumn) GetType 

func (column *RawStringColumn) GetType() ColumnType

func (*RawStringColumn) GetUserConfig 

func (column *RawStringColumn) GetUserConfig() Resource

func (*RawStringColumn) IsRaw 

func (column *RawStringColumn) IsRaw() bool

type Resource 

type Resource interface {
	GetName() string
	GetResourceType() resource.Type
	GetIndex() int
	SetIndex(int)
	GetFilePath() string
	SetFilePath(string)
	GetEmbed() *Embed
	SetEmbed(*Embed)
}

func FindDuplicateResourceName added in v0.2.0 

func FindDuplicateResourceName(resources ...Resource) []Resource

type ResourceConfigFields added in v0.2.0 

type ResourceConfigFields struct {
	Name     string `json:"name" yaml:"name"`
	Index    int    `json:"index" yaml:"-"`
	FilePath string `json:"file_path" yaml:"-"`
	Embed    *Embed `json:"embed" yaml:"-"`
}

func (*ResourceConfigFields) GetEmbed added in v0.2.0 

func (resourceConfigFields *ResourceConfigFields) GetEmbed() *Embed

func (*ResourceConfigFields) GetFilePath added in v0.2.0 

func (resourceConfigFields *ResourceConfigFields) GetFilePath() string

func (*ResourceConfigFields) GetIndex added in v0.2.0 

func (resourceConfigFields *ResourceConfigFields) GetIndex() int

func (*ResourceConfigFields) GetName added in v0.2.0 

func (resourceConfigFields *ResourceConfigFields) GetName() string

func (*ResourceConfigFields) SetEmbed added in v0.2.0 

func (resourceConfigFields *ResourceConfigFields) SetEmbed(embed *Embed)

func (*ResourceConfigFields) SetFilePath added in v0.2.0 

func (resourceConfigFields *ResourceConfigFields) SetFilePath(filePath string)

func (*ResourceConfigFields) SetIndex added in v0.2.0 

func (resourceConfigFields *ResourceConfigFields) SetIndex(index int)

type SparkCompute 

type SparkCompute struct {
	Executors           int32     `json:"executors" yaml:"executors"`
	DriverCPU           Quantity  `json:"driver_cpu" yaml:"driver_cpu"`
	DriverMem           Quantity  `json:"driver_mem" yaml:"driver_mem"`
	DriverMemOverhead   *Quantity `json:"driver_mem_overhead" yaml:"driver_mem_overhead"`
	ExecutorCPU         Quantity  `json:"executor_cpu" yaml:"executor_cpu"`
	ExecutorMem         Quantity  `json:"executor_mem" yaml:"executor_mem"`
	ExecutorMemOverhead *Quantity `json:"executor_mem_overhead" yaml:"executor_mem_overhead"`
	MemOverheadFactor   *float64  `json:"mem_overhead_factor" yaml:"mem_overhead_factor"`
}

func MaxSparkCompute 

func MaxSparkCompute(sparkComputes ...*SparkCompute) *SparkCompute

func (*SparkCompute) ID 

func (sparkCompute *SparkCompute) ID() string

type TFCompute 

type TFCompute struct {
	CPU *Quantity `json:"cpu" yaml:"cpu"`
	Mem *Quantity `json:"mem" yaml:"mem"`
	GPU *int64    `json:"gpu" yaml:"gpu"`
}

func MaxTFCompute 

func MaxTFCompute(tfComputes ...*TFCompute) *TFCompute

func (*TFCompute) ID 

func (tfCompute *TFCompute) ID() string

type Tags 

type Tags map[string]interface{}

func (Tags) ID 

func (tags Tags) ID() string

type Template 

type Template struct {
	ResourceConfigFields
	YAML string `json:"yaml" yaml:"yaml"`
}

func (*Template) GetResourceType 

func (template *Template) GetResourceType() resource.Type

func (*Template) Populate 

func (template *Template) Populate(emb *Embed) (string, error)

func (*Template) VariableNames 

func (template *Template) VariableNames() []string

type Templates 

type Templates []*Template

func (Templates) Map 

func (templates Templates) Map() map[string]*Template

func (Templates) Names 

func (templates Templates) Names() []string

func (Templates) Validate 

func (templates Templates) Validate() error

type TransformedColumn 

type TransformedColumn struct {
	ResourceConfigFields
	Transformer string        `json:"transformer" yaml:"transformer"`
	Inputs      *Inputs       `json:"inputs" yaml:"inputs"`
	Compute     *SparkCompute `json:"compute" yaml:"compute"`
	Tags        Tags          `json:"tags" yaml:"tags"`
}

func (*TransformedColumn) GetResourceType 

func (column *TransformedColumn) GetResourceType() resource.Type

func (*TransformedColumn) InputColumnNames 

func (column *TransformedColumn) InputColumnNames() []string

func (*TransformedColumn) IsRaw 

func (column *TransformedColumn) IsRaw() bool

type TransformedColumns 

type TransformedColumns []*TransformedColumn

func (TransformedColumns) Get 

func (columns TransformedColumns) Get(name string) *TransformedColumn

func (TransformedColumns) Names 

func (columns TransformedColumns) Names() []string

func (TransformedColumns) Validate 

func (columns TransformedColumns) Validate() error

type Transformer 

type Transformer struct {
	ResourceConfigFields
	Inputs     *Inputs    `json:"inputs"  yaml:"inputs"`
	OutputType ColumnType `json:"output_type"  yaml:"output_type"`
	Path       string     `json:"path"  yaml:"path"`
}

func (*Transformer) GetResourceType 

func (transformer *Transformer) GetResourceType() resource.Type

type Transformers 

type Transformers []*Transformer

func (Transformers) Get 

func (transformers Transformers) Get(name string) *Transformer

func (Transformers) Names 

func (transformers Transformers) Names() []string

func (Transformers) Validate 

func (transformers Transformers) Validate() error

type ValueType 

type ValueType int
const (
	UnknownValueType ValueType = iota
	IntegerValueType
	FloatValueType
	StringValueType
	BoolValueType
)

func ValueTypeFromString 

func ValueTypeFromString(s string) ValueType

func (ValueType) MarshalBinary 

func (t ValueType) MarshalBinary() ([]byte, error)

MarshalBinary satisfies BinaryMarshaler

func (ValueType) MarshalText 

func (t ValueType) MarshalText() ([]byte, error)

MarshalText satisfies TextMarshaler

func (ValueType) String 

func (t ValueType) String() string

func (*ValueType) UnmarshalBinary 

func (t *ValueType) UnmarshalBinary(data []byte) error

UnmarshalBinary satisfies BinaryUnmarshaler Needed for msgpack

func (*ValueType) UnmarshalText 

func (t *ValueType) UnmarshalText(text []byte) error

UnmarshalText satisfies TextUnmarshaler

type ValueTypes 

type ValueTypes []ValueType

func (ValueTypes) String 

func (ts ValueTypes) String() string

func (ValueTypes) StringList 

func (ts ValueTypes) StringList() []string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.