ADD attribute decoder config

appinterface/tendermint/blockresulteventattrivutedecoder.go

@@ -0,0 +1,6 @@
+package tendermint
+
+type BlockResultEventAttributeDecoder interface {
+	DecodeKey(string) (string, error)
+	DecodeValue(string) (string, error)
+}

appinterface/tendermint/client.go

@@ -9,6 +9,6 @@ type Client interface {
 	Genesis() (*genesis.Genesis, error)
 	GenesisChunked() (*genesis.Genesis, error)
 	Block(height int64) (*usecase_model.Block, *usecase_model.RawBlock, error)
-	BlockResults(height int64) (*usecase_model.BlockResults, error)
+	BlockResults(height int64, decoder BlockResultEventAttributeDecoder) (*usecase_model.BlockResults, error)
 	LatestBlockHeight() (int64, error)
 }

bootstrap/config/config.go

@@ -16,14 +16,15 @@ type Config struct {
 }
 
 type IndexService struct {
-	Enable                     bool                       `yaml:"enable" toml:"enable" xml:"enable" json:"enable,omitempty"`
-	StartingBlockHeight        int64                      `yaml:"starting_block_height" toml:"starting_block_height" xml:"starting_block_height" json:"starting_block_height,omitempty"`
-	Mode                       string                     `yaml:"mode" toml:"mode" xml:"mode" json:"mode,omitempty"`
-	WindowSize                 int                        `yaml:"window_size" toml:"window_size" xml:"window_size" json:"window_size,omitempty"`
-	Projection                 Projection                 `yaml:"projection" toml:"projection" xml:"projection" json:"projection"`
-	CronJob                    CronJob                    `yaml:"cron_job" toml:"cron_job" xml:"cron_job" json:"cron_job"`
-	CosmosVersionEnabledHeight CosmosVersionEnabledHeight `yaml:"cosmos_version_enabled_height" toml:"cosmos_version_enabled_height" xml:"cosmos_version_enabled_height" json:"cosmos_version_enabled_height"`
-	GithubAPI                  GithubAPI                  `yaml:"github_api" toml:"github_api" xml:"github_api" json:"github_api"`
+	Enable                                bool                       `yaml:"enable" toml:"enable" xml:"enable" json:"enable,omitempty"`
+	StartingBlockHeight                   int64                      `yaml:"starting_block_height" toml:"starting_block_height" xml:"starting_block_height" json:"starting_block_height,omitempty"`
+	Mode                                  string                     `yaml:"mode" toml:"mode" xml:"mode" json:"mode,omitempty"`
+	WindowSize                            int                        `yaml:"window_size" toml:"window_size" xml:"window_size" json:"window_size,omitempty"`
+	Projection                            Projection                 `yaml:"projection" toml:"projection" xml:"projection" json:"projection"`
+	CronJob                               CronJob                    `yaml:"cron_job" toml:"cron_job" xml:"cron_job" json:"cron_job"`
+	CosmosVersionEnabledHeight            CosmosVersionEnabledHeight `yaml:"cosmos_version_enabled_height" toml:"cosmos_version_enabled_height" xml:"cosmos_version_enabled_height" json:"cosmos_version_enabled_height"`
+	GithubAPI                             GithubAPI                  `yaml:"github_api" toml:"github_api" xml:"github_api" json:"github_api"`
+	BlockResultEventAttributeDecodeMethod string                     `yaml:"block_result_event_attribute_decode_method" toml:"block_result_event_attribute_decode_method" xml:"block_result_event_attribute_decode_method" json:"block_result_event_attribute_decode_method"`
 }
 
 type HTTPService struct {

bootstrap/indexservice.go

@@ -22,17 +22,18 @@ type IndexService struct {
 	projections []projection_entity.Projection
 	cronJobs    []projection_entity.CronJob
 
-	mode                     string
-	accountAddressPrefix     string
-	consNodeAddressPrefix    string
-	bondingDenom             string
-	windowSize               int
-	tendermintHTTPRPCURL     string
-	cosmosAppHTTPRPCURL      string
-	insecureTendermintClient bool
-	insecureCosmosAppClient  bool
-	strictGenesisParsing     bool
-	startingBlockHeight      int64
+	mode                                  string
+	accountAddressPrefix                  string
+	consNodeAddressPrefix                 string
+	bondingDenom                          string
+	windowSize                            int
+	tendermintHTTPRPCURL                  string
+	cosmosAppHTTPRPCURL                   string
+	insecureTendermintClient              bool
+	insecureCosmosAppClient               bool
+	strictGenesisParsing                  bool
+	startingBlockHeight                   int64
+	BlockResultEventAttributeDecodeMethod string
 
 	cosmosVersionBlockHeight utils.CosmosVersionBlockHeight
 
@@ -57,17 +58,18 @@ func NewIndexService(
 		projections: projections,
 		cronJobs:    cronJobs,
 
-		mode:                     config.IndexService.Mode,
-		consNodeAddressPrefix:    config.Blockchain.ConNodeAddressPrefix,
-		accountAddressPrefix:     config.Blockchain.AccountAddressPrefix,
-		bondingDenom:             config.Blockchain.BondingDenom,
-		windowSize:               config.IndexService.WindowSize,
-		tendermintHTTPRPCURL:     config.TendermintApp.HTTPRPCUrl,
-		cosmosAppHTTPRPCURL:      config.CosmosApp.HTTPRPCUrl,
-		insecureTendermintClient: config.TendermintApp.Insecure,
-		insecureCosmosAppClient:  config.CosmosApp.Insecure,
-		strictGenesisParsing:     config.TendermintApp.StrictGenesisParsing,
-		startingBlockHeight:      config.IndexService.StartingBlockHeight,
+		mode:                                  config.IndexService.Mode,
+		consNodeAddressPrefix:                 config.Blockchain.ConNodeAddressPrefix,
+		accountAddressPrefix:                  config.Blockchain.AccountAddressPrefix,
+		bondingDenom:                          config.Blockchain.BondingDenom,
+		windowSize:                            config.IndexService.WindowSize,
+		tendermintHTTPRPCURL:                  config.TendermintApp.HTTPRPCUrl,
+		cosmosAppHTTPRPCURL:                   config.CosmosApp.HTTPRPCUrl,
+		insecureTendermintClient:              config.TendermintApp.Insecure,
+		insecureCosmosAppClient:               config.CosmosApp.Insecure,
+		strictGenesisParsing:                  config.TendermintApp.StrictGenesisParsing,
+		startingBlockHeight:                   config.IndexService.StartingBlockHeight,
+		BlockResultEventAttributeDecodeMethod: config.IndexService.BlockResultEventAttributeDecodeMethod,
 		cosmosVersionBlockHeight: utils.CosmosVersionBlockHeight{
 			V0_42_7: utils.ParserBlockHeight(config.IndexService.CosmosVersionEnabledHeight.V0_42_7),
 		},
@@ -144,15 +146,16 @@ func (service *IndexService) RunEventStoreMode() error {
 			Logger:  service.logger,
 			RDbConn: service.rdbConn,
 			Config: SyncManagerConfig{
-				WindowSize:               service.windowSize,
-				TendermintRPCUrl:         service.tendermintHTTPRPCURL,
-				CosmosAppHTTPRPCURL:      service.cosmosAppHTTPRPCURL,
-				InsecureTendermintClient: service.insecureTendermintClient,
-				InsecureCosmosAppClient:  service.insecureCosmosAppClient,
-				StrictGenesisParsing:     service.strictGenesisParsing,
-				AccountAddressPrefix:     service.accountAddressPrefix,
-				StakingDenom:             service.bondingDenom,
-				StartingBlockHeight:      service.startingBlockHeight,
+				WindowSize:                            service.windowSize,
+				TendermintRPCUrl:                      service.tendermintHTTPRPCURL,
+				CosmosAppHTTPRPCURL:                   service.cosmosAppHTTPRPCURL,
+				InsecureTendermintClient:              service.insecureTendermintClient,
+				InsecureCosmosAppClient:               service.insecureCosmosAppClient,
+				StrictGenesisParsing:                  service.strictGenesisParsing,
+				AccountAddressPrefix:                  service.accountAddressPrefix,
+				StakingDenom:                          service.bondingDenom,
+				StartingBlockHeight:                   service.startingBlockHeight,
+				BlockResultEventAttributeDecodeMethod: service.BlockResultEventAttributeDecodeMethod,
 			},
 			TxDecoder: service.txDecoder,
 		},

bootstrap/syncmanager.go

@@ -7,6 +7,7 @@ import (
 	"github.com/cenkalti/backoff/v4"
 	cosmosapp_interface "github.com/crypto-com/chain-indexing/appinterface/cosmosapp"
 	eventhandler_interface "github.com/crypto-com/chain-indexing/appinterface/eventhandler"
+	tendermint_interface "github.com/crypto-com/chain-indexing/appinterface/tendermint"
 	"github.com/crypto-com/chain-indexing/external/txdecoder"
 	cosmosapp_infrastructure "github.com/crypto-com/chain-indexing/infrastructure/cosmosapp"
 	"github.com/crypto-com/chain-indexing/usecase/model"
@@ -54,6 +55,8 @@ type SyncManager struct {
 	startingBlockHeight int64
 
 	txDecoder txdecoder.TxDecoder
+
+	eventAttributeDecoder tendermint_interface.BlockResultEventAttributeDecoder
 }
 
 type SyncManagerParams struct {
@@ -65,15 +68,16 @@ type SyncManagerParams struct {
 }
 
 type SyncManagerConfig struct {
-	WindowSize               int
-	TendermintRPCUrl         string
-	CosmosAppHTTPRPCURL      string
-	InsecureTendermintClient bool
-	InsecureCosmosAppClient  bool
-	StrictGenesisParsing     bool
-	AccountAddressPrefix     string
-	StakingDenom             string
-	StartingBlockHeight      int64
+	WindowSize                            int
+	TendermintRPCUrl                      string
+	CosmosAppHTTPRPCURL                   string
+	InsecureTendermintClient              bool
+	InsecureCosmosAppClient               bool
+	StrictGenesisParsing                  bool
+	AccountAddressPrefix                  string
+	StakingDenom                          string
+	StartingBlockHeight                   int64
+	BlockResultEventAttributeDecodeMethod string
 }
 
 // NewSyncManager creates a new feed with polling for latest block starts at a specific height
@@ -106,6 +110,15 @@ func NewSyncManager(
 		)
 	}
 
+	var eventAttributeDecoder tendermint_interface.BlockResultEventAttributeDecoder
+
+	switch params.Config.BlockResultEventAttributeDecodeMethod {
+	case "base64":
+		eventAttributeDecoder = &tendermint.Base64BlockResultEventAttributeDecoder{}
+	default:
+		eventAttributeDecoder = &tendermint.RawBlockResultEventAttributeDecoder{}
+	}
+
 	return &SyncManager{
 		rdbConn:          params.RDbConn,
 		tendermintClient: tendermintClient,
@@ -132,6 +145,8 @@ func NewSyncManager(
 		startingBlockHeight: params.Config.StartingBlockHeight,
 
 		txDecoder: params.TxDecoder,
+
+		eventAttributeDecoder: eventAttributeDecoder,
 	}
 }
 
@@ -241,7 +256,7 @@ func (manager *SyncManager) syncBlockWorker(blockHeight int64) ([]command_entity
 		return nil, fmt.Errorf("error requesting chain block at height %d: %v", blockHeight, err)
 	}
 
-	blockResults, err := manager.tendermintClient.BlockResults(blockHeight)
+	blockResults, err := manager.tendermintClient.BlockResults(blockHeight, manager.eventAttributeDecoder)
 	if err != nil {
 		return nil, fmt.Errorf("error requesting chain block_results at height %d: %v", blockHeight, err)
 	}

example/go.sum

@@ -482,6 +482,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/crypto-com/chain-indexing v1.2.1-0.20220111132225-b2712ba4fe23 h1:Pit1X4i8SXU/pUg60mo3tj/fKM+xYJ5j3TGtBK41D3k=
 github.com/crypto-com/chain-indexing v1.2.1-0.20220111132225-b2712ba4fe23/go.mod h1:0KBBR7C2RZDavMpCBKlNtHkMZZ7tyndioZQRWUTD4YI=
+github.com/crypto-com/chain-indexing v1.3.0/go.mod h1:0KBBR7C2RZDavMpCBKlNtHkMZZ7tyndioZQRWUTD4YI=
 github.com/crypto-org-chain/chain-main/v3 v3.0.0-croeseid h1:YAffq+tYiSqYXgIb11Vc6dvBtgSvRsf2g1wPaMFkQUA=
 github.com/crypto-org-chain/chain-main/v3 v3.0.0-croeseid/go.mod h1:92Z70bDbsScrWzIHB496p7nXKydtq5am9rqa0fCpbM8=
 github.com/crypto-org-chain/cronos v0.6.0-testnet h1:iFLwra4QMZ6HgIEB1P0vMCa9pnQTeXjwSuYbH4glxqA=

infrastructure/tendermint/blockresulteventattrivutedecoder.go

@@ -0,0 +1,35 @@
+package tendermint
+
+import "encoding/base64"
+
+type RawBlockResultEventAttributeDecoder struct {
+}
+
+func (decoder RawBlockResultEventAttributeDecoder) DecodeKey(s string) (string, error) {
+	return s, nil
+}
+
+func (decoder RawBlockResultEventAttributeDecoder) DecodeValue(s string) (string, error) {
+	return s, nil
+}
+
+type Base64BlockResultEventAttributeDecoder struct {
+}
+
+func (decoder Base64BlockResultEventAttributeDecoder) DecodeKey(s string) (string, error) {
+	decoded, err := base64.StdEncoding.DecodeString(s)
+	if err != nil {
+		return "", err
+	}
+
+	return string(decoded), nil
+}
+
+func (decoder Base64BlockResultEventAttributeDecoder) DecodeValue(s string) (string, error) {
+	decoded, err := base64.StdEncoding.DecodeString(s)
+	if err != nil {
+		return "", err
+	}
+
+	return string(decoded), nil
+}

infrastructure/tendermint/httpclient.go

@@ -164,7 +164,7 @@ func (client *HTTPClient) Block(height int64) (*usecase_model.Block, *usecase_mo
 	return block, rawBlock, nil
 }
 
-func (client *HTTPClient) BlockResults(height int64) (*usecase_model.BlockResults, error) {
+func (client *HTTPClient) BlockResults(height int64, eventAttributeDecoder tendermint.BlockResultEventAttributeDecoder) (*usecase_model.BlockResults, error) {
 	var err error
 
 	rawRespBody, err := client.request("block_results", queryKV{
@@ -176,7 +176,7 @@ func (client *HTTPClient) BlockResults(height int64) (*usecase_model.BlockResult
 	}
 	defer rawRespBody.Close()
 
-	blockResults, err := ParseBlockResultsResp(rawRespBody)
+	blockResults, err := ParseBlockResultsResp(rawRespBody, eventAttributeDecoder)
 	if err != nil {
 		return nil, err
 	}

infrastructure/tendermint/parser.go

@@ -3,9 +3,9 @@ package tendermint
 import (
 	"encoding/base64"
 	"fmt"
+	tendermint_interface "github.com/crypto-com/chain-indexing/appinterface/tendermint"
 	"io"
 	"math/big"
-	"regexp"
 	"strconv"
 	"strings"
 
@@ -17,10 +17,6 @@ import (
 	jsoniter "github.com/json-iterator/go"
 )
 
-var (
-	regxASCII = regexp.MustCompile("^[\x00-\x7F]+$")
-)
-
 // Block related parsing functions
 func ParseGenesisResp(rawRespReader io.Reader, strictParsing bool) (*genesis.Genesis, error) {
 	var genesisResp GenesisResp
@@ -105,7 +101,7 @@ func parseBlockSignatures(rawSignatures []model.RawBlockSignature) []model.Block
 }
 
 // RawBlockResults related parsing functions
-func ParseBlockResultsResp(rawRespReader io.Reader) (*model.BlockResults, error) {
+func ParseBlockResultsResp(rawRespReader io.Reader, eventAttributeDecoder tendermint_interface.BlockResultEventAttributeDecoder) (*model.BlockResults, error) {
 	var err error
 
 	var resp RawBlockResultsResp
@@ -122,21 +118,21 @@ func ParseBlockResultsResp(rawRespReader io.Reader) (*model.BlockResults, error)
 		return nil, fmt.Errorf("error converting block height to unsigned integer: %v", err)
 	}
 
-	txsResults := parseBlockResultsTxsResults(rawBlockResults.TxsResults)
+	txsResults := parseBlockResultsTxsResults(rawBlockResults.TxsResults, eventAttributeDecoder)
 	return &model.BlockResults{
 		Height:                int64(height),
 		TxsResults:            txsResults,
-		BeginBlockEvents:      parseBlockResultsEvents(rawBlockResults.BeginBlockEvents),
-		EndBlockEvents:        parseBlockResultsEvents(rawBlockResults.EndBlockEvents),
+		BeginBlockEvents:      parseBlockResultsEvents(rawBlockResults.BeginBlockEvents, eventAttributeDecoder),
+		EndBlockEvents:        parseBlockResultsEvents(rawBlockResults.EndBlockEvents, eventAttributeDecoder),
 		ValidatorUpdates:      parseBlockResultsValidatorUpdates(rawBlockResults.ValidatorUpdates),
 		ConsensusParamUpdates: parseBlockResultsConsensusParamsUpdates(rawBlockResults.ConsensusParamUpdates),
 	}, nil
 }
 
-func parseBlockResultsTxsResults(rawTxsResults []RawBlockResultsTxsResult) []model.BlockResultsTxsResult {
+func parseBlockResultsTxsResults(rawTxsResults []RawBlockResultsTxsResult, eventAttributeDecoder tendermint_interface.BlockResultEventAttributeDecoder) []model.BlockResultsTxsResult {
 	txsResults := make([]model.BlockResultsTxsResult, 0, len(rawTxsResults))
 	for _, rawTxsResult := range rawTxsResults {
-		events := parseBlockResultsEvents(rawTxsResult.Events)
+		events := parseBlockResultsEvents(rawTxsResult.Events, eventAttributeDecoder)
 
 		txsResults = append(txsResults, model.BlockResultsTxsResult{
 			Code:      rawTxsResult.Code,
@@ -202,7 +198,7 @@ func parseBlockResultsTxsResultLogEvents(rawEvents []RawBlockResultsEvent) []mod
 	return events
 }
 
-func parseBlockResultsEvents(rawEvents []RawBlockResultsEvent) []model.BlockResultsEvent {
+func parseBlockResultsEvents(rawEvents []RawBlockResultsEvent, eventAttributeDecoder tendermint_interface.BlockResultEventAttributeDecoder) []model.BlockResultsEvent {
 	if rawEvents == nil {
 		return []model.BlockResultsEvent{}
 	}
@@ -211,13 +207,13 @@ func parseBlockResultsEvents(rawEvents []RawBlockResultsEvent) []model.BlockResu
 	for _, rawEvent := range rawEvents {
 		attributes := make([]model.BlockResultsEventAttribute, 0, len(rawEvent.Attributes))
 		for _, rawAttribute := range rawEvent.Attributes {
-			key, err := base64Decode(rawAttribute.Key)
-			if err != nil || !regxASCII.MatchString(key) {
-				key = rawAttribute.Key
+			key, err := eventAttributeDecoder.DecodeKey(rawAttribute.Key)
+			if err != nil {
+				panic(fmt.Sprintf("error blcok result event %s attribute key (%s): %v", rawEvent.Type, rawAttribute.Key, err))
 			}
-			value, err := base64Decode(rawAttribute.Value)
-			if err != nil || !regxASCII.MatchString(value) {
-				value = rawAttribute.Value
+			value, err := eventAttributeDecoder.DecodeValue(rawAttribute.Value)
+			if err != nil {
+				panic(fmt.Sprintf("error blcok result event %s attribute key (%s): %v", rawEvent.Type, rawAttribute.Key, err))
 			}
 			attributes = append(attributes, model.BlockResultsEventAttribute{
 				Key:   key,

usecase/parser/begin_block_events_test.go

@@ -1,6 +1,7 @@
 package parser_test
 
 import (
+	"github.com/crypto-com/chain-indexing/infrastructure/tendermint"
 	"github.com/crypto-com/chain-indexing/usecase/model"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -15,7 +16,7 @@ import (
 var _ = Describe("ParseBeginBlockEventsCommands", func() {
 	Describe("MsgSend", func() {
 		It("should return commands corresponding to events in begin_block_events", func() {
-			blockResults := mustParseBlockResultsResp(usecase_parser_test.BEGIN_BLOCK_COMMON_EVENTS_BLOCK_RESULTS_RESP)
+			blockResults := mustParseBlockResultsResp(usecase_parser_test.BEGIN_BLOCK_COMMON_EVENTS_BLOCK_RESULTS_RESP, &tendermint.Base64BlockResultEventAttributeDecoder{})
 			block, _ := mustParseBlockResp(usecase_parser_test.BEGIN_BLOCK_COMMON_EVENTS_BLOCK_RESP)
 
 			bondingDenom := "basetcro"
@@ -90,7 +91,7 @@ var _ = Describe("ParseBeginBlockEventsCommands", func() {
 		})
 
 		It("should return ValidatorSlashed and ValidatorJailed command base on missing signature events", func() {
-			blockResults := mustParseBlockResultsResp(usecase_parser_test.BEGIN_BLOCK_SLASH_MISSING_SIGNATURES_EVENT_BLOCK_RESULTS_RESP)
+			blockResults := mustParseBlockResultsResp(usecase_parser_test.BEGIN_BLOCK_SLASH_MISSING_SIGNATURES_EVENT_BLOCK_RESULTS_RESP, &tendermint.Base64BlockResultEventAttributeDecoder{})
 			block, _ := mustParseBlockResp(usecase_parser_test.BEGIN_BLOCK_COMMON_EVENTS_BLOCK_RESP)
 
 			bondingDenom := "basetcro"
@@ -141,7 +142,7 @@ var _ = Describe("ParseBeginBlockEventsCommands", func() {
 		})
 
 		It("should return ValidatorSlashed and ValidatorJailed command base on double sign events", func() {
-			blockResults := mustParseBlockResultsResp(usecase_parser_test.BEGIN_BLOCK_SLASH_DOUBLE_SIGN_EVENT_BLOCK_RESULTS_RESP)
+			blockResults := mustParseBlockResultsResp(usecase_parser_test.BEGIN_BLOCK_SLASH_DOUBLE_SIGN_EVENT_BLOCK_RESULTS_RESP, &tendermint.Base64BlockResultEventAttributeDecoder{})
 			block, _ := mustParseBlockResp(usecase_parser_test.BEGIN_BLOCK_COMMON_EVENTS_BLOCK_RESP)
 
 			bondingDenom := "basetcro"