Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix/units and allowed types #126

Merged
merged 3 commits into from
May 17, 2024
Merged

Fix/units and allowed types #126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0fe1348
fix: Fixed allowed types for IP and MAC
wwoytenko May 17, 2024
5766837
fix: Renamed Email to RandomEmail. Removed some faker transformers
wwoytenko May 17, 2024
3fca5d7
Merge remote-tracking branch 'origin_public/main' into fix/units_and_…
wwoytenko May 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/db/postgres/transformers/email.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ var emailTransformerAllowedChars = []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijk

var EmailTransformerDefinition = utils.NewTransformerDefinition(
utils.NewTransformerProperties(
"Email",
"RandomEmail",
"Generate random email",
),

Expand Down
28 changes: 0 additions & 28 deletions internal/db/postgres/transformers/random_faker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,6 @@ var FakerTransformersDes = map[string]*FakerTransformerDef{
Description: "Generates a random longitude value.",
},

// Faker Datetime
"RandomUnixTime": {
Generator: func(opts ...options.OptionFunc) string {
return fmt.Sprintf("%d", faker.UnixTime())
},
SupportedTypes: []string{"int4", "int8", "numeric"},
Description: "Generates a random Unix timestamp.",
},
"RandomMonthName": {
Generator: faker.MonthName,
SupportedTypes: []string{"text", "varchar"},
Expand Down Expand Up @@ -90,16 +82,6 @@ var FakerTransformersDes = map[string]*FakerTransformerDef{
},

// Faker Internet
"RandomEmail": {
Generator: faker.Email,
SupportedTypes: []string{"text", "varchar"},
Description: "Generates a random email address.",
},
"RandomMacAddress": {
Generator: faker.MacAddress,
SupportedTypes: []string{"text", "varchar", "macaddr", "macaddr8"},
Description: "Generates a random MAC address.",
},
"RandomDomainName": {
Generator: faker.DomainName,
SupportedTypes: []string{"text", "varchar"},
Expand All @@ -115,16 +97,6 @@ var FakerTransformersDes = map[string]*FakerTransformerDef{
SupportedTypes: []string{"text", "varchar"},
Description: "Generates a random username.",
},
"RandomIPv4": {
Generator: faker.IPv4,
SupportedTypes: []string{"text", "varchar", "inet"},
Description: "Generates a random IPv4 address.",
},
"RandomIPv6": {
Generator: faker.IPv6,
SupportedTypes: []string{"text", "varchar", "inet"},
Description: "Generates a random IPv6 address.",
},
"RandomPassword": {
Generator: faker.Password,
SupportedTypes: []string{"text", "varchar"},
Expand Down
6 changes: 3 additions & 3 deletions internal/db/postgres/transformers/random_ip.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ var RandomIpDefinition = utils.NewTransformerDefinition(
SetCastDbType("cidr").
SetDynamicMode(
toolkit.NewDynamicModeProperties().
SetCompatibleTypes("cidr", "text", "varchar"),
SetCompatibleTypes("text", "varchar", "cidr"),
),

engineParameterDefinition,
Expand Down Expand Up @@ -146,8 +146,8 @@ func (rbt *RandomIp) Transform(ctx context.Context, r *toolkit.Record) (*toolkit
return nil, fmt.Errorf("unable to transform value: %w", err)
}

newRawValue := toolkit.NewRawValue([]byte(ipVal.String()), false)
if err = r.SetRawColumnValueByIdx(rbt.columnIdx, newRawValue); err != nil {
newVal := toolkit.NewRawValue([]byte(ipVal.String()), false)
if err = r.SetRawColumnValueByIdx(rbt.columnIdx, newVal); err != nil {
return nil, fmt.Errorf("unable to set new value: %w", err)
}
return r, nil
Expand Down
22 changes: 20 additions & 2 deletions internal/db/postgres/transformers/random_ip_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@ package transformers

import (
"context"
"testing"

"github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils"
"github.com/greenmaskio/greenmask/pkg/toolkit"
"github.com/stretchr/testify/require"
"testing"
)

func TestRandomIpTransformer_Transform_random_dynamic(t *testing.T) {
Expand All @@ -19,7 +20,24 @@ func TestRandomIpTransformer_Transform_random_dynamic(t *testing.T) {
expected string
}{
{
name: "IPv4 dynamic test",
name: "IPv4 dynamic test with strict types",
columnName: "ip_address",
record: map[string]*toolkit.RawValue{
"ip_address": toolkit.NewRawValue([]byte("192.168.1.10"), false),
"net_mask": toolkit.NewRawValue([]byte("192.168.1.0/30"), false),
},
params: map[string]toolkit.ParamsValue{
"engine": toolkit.ParamsValue("random"),
},
dynamicParams: map[string]*toolkit.DynamicParamValue{
"subnet": {
Column: "net_mask",
},
},
expected: "192.168.1.[1,2]",
},
{
name: "IPv4 dynamic test with strings",
columnName: "data",
record: map[string]*toolkit.RawValue{
"data": toolkit.NewRawValue([]byte("192.168.1.10"), false),
Expand Down
12 changes: 9 additions & 3 deletions internal/db/postgres/transformers/random_mac.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils"
"github.com/greenmaskio/greenmask/internal/generators/transformers"
"github.com/greenmaskio/greenmask/pkg/toolkit"
"github.com/jackc/pgx/v5/pgtype"
)

const (
Expand All @@ -46,7 +47,7 @@ var RandomMacAddressDefinition = utils.NewTransformerDefinition(
"Column name",
).SetIsColumn(toolkit.NewColumnProperties().
SetAffected(true).
SetAllowedColumnTypes("macaddr"),
SetAllowedColumnTypes("macaddr", "varchar", "text"),
).SetRequired(true),

toolkit.MustNewParameterDefinition(
Expand Down Expand Up @@ -172,7 +173,11 @@ func (rbt *RandomMac) Done(ctx context.Context) error {

func (rbt *RandomMac) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error) {

_, err := r.ScanColumnValueByIdx(rbt.columnIdx, &rbt.originalMac)
rawVal, err := r.GetRawColumnValueByIdx(rbt.columnIdx)
if err != nil {
return nil, fmt.Errorf("unable to scan value: %w", err)
}
err = r.Driver.ScanValueByTypeOid(pgtype.MacaddrOID, rawVal.Data, &rbt.originalMac)
if err != nil {
return nil, fmt.Errorf("unable to scan value: %w", err)
}
Expand All @@ -182,7 +187,8 @@ func (rbt *RandomMac) Transform(ctx context.Context, r *toolkit.Record) (*toolki
return nil, fmt.Errorf("unable to transform value: %w", err)
}

if err = r.SetColumnValueByIdx(rbt.columnIdx, macAddr); err != nil {
newVal := toolkit.NewRawValue([]byte(macAddr.String()), false)
if err = r.SetRawColumnValueByIdx(rbt.columnIdx, newVal); err != nil {
return nil, fmt.Errorf("unable to set new value: %w", err)
}

Expand Down
24 changes: 18 additions & 6 deletions internal/db/postgres/transformers/random_mac_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,17 @@ func TestRandomMacTransformer_Transform_random(t *testing.T) {
managementType: managementTypeNameLocal,
castType: castTypeNameGroup,
},
{
name: "Random mac addr in text without keepOriginalVendor with Universal and Individual",
columnName: "data",
original: "03:1a:2b:3c:4d:5e",
params: map[string]toolkit.ParamsValue{
"engine": toolkit.ParamsValue("hash"),
"keep_original_vendor": toolkit.ParamsValue("false"),
},
managementType: managementTypeNameLocal,
castType: castTypeNameGroup,
},
}

for _, tt := range tests {
Expand Down Expand Up @@ -130,18 +141,19 @@ func TestRandomMacTransformer_Transform_random(t *testing.T) {
)
require.NoError(t, err)
var res net.HardwareAddr
isNull, err := r.ScanColumnValueByName(tt.columnName, &res)
rawVal, err := r.GetRawColumnValueByName(tt.columnName)
require.NoError(t, err)
require.False(t, rawVal.IsNull)
require.NotEmpty(t, rawVal.Data)
err = r.Driver.ScanValueByTypeName("macaddr", rawVal.Data, &res)
require.NoError(t, err)
require.False(t, isNull)

newMacAddrInfo, err := transformers.ExploreMacAddress(res)
require.NoError(t, err)

if string(tt.params["keep_original_vendor"]) == "true" {
require.Equal(t, tt.original[:8], res.String()[0:8])
}

if tt.castType != castTypeNameAny && tt.managementType != managementTypeNameAny {
require.Equal(t, tt.original[:8], res.String()[:8])
} else if tt.castType != castTypeNameAny && tt.managementType != managementTypeNameAny {
require.True(
t,
castTypeNameToIndex(tt.castType) == newMacAddrInfo.CastType && managementTypeNameToIndex(tt.managementType) == newMacAddrInfo.ManagementType,
Expand Down
17 changes: 11 additions & 6 deletions internal/db/postgres/transformers/random_unix_timestamp.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,16 @@ import (
)

const (
secondsUnit = "sec"
milliUnit = "milli"
microUnit = "micro"
nanoUnit = "nano"
secondsUnit = "second"
milliUnit = "millisecond"
microUnit = "microsecond"
nanoUnit = "nanosecond"
)

var timestampUnitValues = []string{
secondsUnit, milliUnit, microUnit, nanoUnit,
}

var unixTimestampTransformerDefinition = utils.NewTransformerDefinition(
utils.NewTransformerProperties(
"RandomUnixTimestamp",
Expand Down Expand Up @@ -219,12 +223,13 @@ func getUnixByUnit(v time.Time, unit string) int64 {

func validateDateUnitParameterValue(p *toolkit.ParameterDefinition, v toolkit.ParamsValue) (toolkit.ValidationWarnings, error) {

if !slices.Contains([]string{secondsUnit, milliUnit, microUnit, nanoUnit}, string(v)) {
if !slices.Contains(timestampUnitValues, string(v)) {
return toolkit.ValidationWarnings{
toolkit.NewValidationWarning().
SetSeverity(toolkit.ErrorValidationSeverity).
AddMeta("ParameterValue", string(v)).
SetMsg("wrong truncation part value: must be one of nano, second, minute, hour, day, month, year"),
AddMeta("AllowedValues", truncateParts).
SetMsg("wrong timestamp unit value"),
}, nil
}
return nil, nil
Expand Down
16 changes: 16 additions & 0 deletions pkg/toolkit/tesing_helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,22 @@ var columnList = []*Column{
NotNull: false,
Length: -1,
},
{
Name: "ip_address",
TypeName: "inet",
TypeOid: pgtype.InetOID,
Num: 22,
NotNull: false,
Length: -1,
},
{
Name: "net_mask",
TypeName: "cidr",
TypeOid: pgtype.CIDROID,
Num: 23,
NotNull: false,
Length: -1,
},
}

// GetDriverAndRecord - return adhoc table for testing
Expand Down