From 708759a4b50edaea1711ce65fa8c142d902251bb Mon Sep 17 00:00:00 2001 From: Chengxun Lee <24319042+bclswl0827@users.noreply.github.com> Date: Wed, 20 Mar 2024 12:38:44 +0800 Subject: [PATCH] Add earthquake event source API support of CEA and INGV --- CHANGELOG.md | 8 +- VERSION | 2 +- app/v1/trace/cea.go | 154 +++++++++++++++++ app/v1/trace/ingv.go | 157 ++++++++++++++++++ app/v1/trace/module.go | 18 +- driver/dao/insert.go | 10 ++ driver/dao/migrate.go | 7 + driver/dao/{operation.go => query.go} | 8 - driver/seedlink/end.go | 16 +- frontend/dist/asset-manifest.json | 4 +- frontend/dist/index.html | 2 +- .../js/{main.6c81bc75.js => main.6f5b5049.js} | 4 +- ...CENSE.txt => main.6f5b5049.js.LICENSE.txt} | 0 frontend/src/.env | 4 +- publisher/array.go | 32 +--- publisher/decode.go | 30 ++++ publisher/encode.go | 17 ++ utils/request/get.go | 2 +- utils/request/post.go | 2 +- 19 files changed, 411 insertions(+), 66 deletions(-) create mode 100644 app/v1/trace/cea.go create mode 100644 app/v1/trace/ingv.go create mode 100644 driver/dao/insert.go create mode 100644 driver/dao/migrate.go rename driver/dao/{operation.go => query.go} (68%) rename frontend/dist/static/js/{main.6c81bc75.js => main.6f5b5049.js} (99%) rename frontend/dist/static/js/{main.6c81bc75.js.LICENSE.txt => main.6f5b5049.js.LICENSE.txt} (100%) create mode 100644 publisher/decode.go create mode 100644 publisher/encode.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 57673793..32b11096 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,15 @@ Starting from v2.2.5, all notable changes to this project will be documented in this file. +## v2.11.8 + +- Add earthquake event source API support of CEA and INGV +- Reuse of int32 array encoding and decoding functions +- Specify the minimun TLS version to 1.0 in HTTP client + ## v2.11.7 -- Support earthquake event source API of Korea Meteorological Administration +- Add earthquake event source API support of KMA ## v2.11.6 diff --git a/VERSION b/VERSION index cee6f277..4485bed5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v2.11.7 +v2.11.8 diff --git a/app/v1/trace/cea.go b/app/v1/trace/cea.go new file mode 100644 index 00000000..70c0ae0a --- /dev/null +++ b/app/v1/trace/cea.go @@ -0,0 +1,154 @@ +package trace + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/anyshake/observer/utils/duration" + "github.com/anyshake/observer/utils/request" +) + +type CEA_DASE struct { + DataSourceCache +} + +func (c *CEA_DASE) Property() string { + return "Commissariat à l'Energie Atomique" +} + +func (c *CEA_DASE) Fetch() ([]byte, error) { + if duration.Difference(time.Now(), c.Time) <= EXPIRATION { + return c.Cache, nil + } + + res, err := request.GET( + "https://www-dase.cea.fr/evenement/derniers_evenements.php", + 10*time.Second, time.Second, 3, false, nil, + ) + if err != nil { + return nil, err + } + + c.Time = time.Now() + c.Cache = make([]byte, len(res)) + copy(c.Cache, res) + + return res, nil +} + +func (c *CEA_DASE) Parse(data []byte) (map[string]any, error) { + result := make(map[string]any) + result["data"] = make([]any, 0) + + reader := bytes.NewBuffer(data) + doc, err := goquery.NewDocumentFromReader(reader) + if err != nil { + return nil, err + } + + doc.Find(".arial11bleu").Each(func(i int, s *goquery.Selection) { + item := make(map[string]any) + var dateString string + s.Find("td").Each(func(i int, s *goquery.Selection) { + value := strings.TrimSpace(s.Text()) + switch i { + case 0: + item["depth"] = -1 + dateString = value + case 1: + item["timestamp"] = c.getTimestamp(fmt.Sprintf("%s %s", dateString, value)) + case 2: + item["latitude"] = c.getLatitude(value) + item["longitude"] = c.getLongitude(value) + case 3: + item["event"] = value + item["region"] = value + case 4: + item["magnitude"] = c.getMagnitude(value) + } + }) + result["data"] = append(result["data"].([]any), item) + }) + + return result, nil +} + +func (c *CEA_DASE) Format(latitude, longitude float64, data map[string]any) ([]Event, error) { + var list []Event + for _, v := range data["data"].([]any) { + l := Event{ + Verfied: false, + Latitude: v.(map[string]any)["latitude"].(float64), + Longitude: v.(map[string]any)["longitude"].(float64), + Depth: float64(v.(map[string]any)["depth"].(int)), + Event: v.(map[string]any)["event"].(string), + Region: v.(map[string]any)["region"].(string), + Timestamp: v.(map[string]any)["timestamp"].(int64), + Magnitude: v.(map[string]any)["magnitude"].(float64), + } + l.Distance = getDistance(latitude, l.Latitude, longitude, l.Longitude) + l.Estimation = getEstimation(l.Depth, l.Distance) + + list = append(list, l) + } + + return list, nil +} + +func (c *CEA_DASE) List(latitude, longitude float64) ([]Event, error) { + res, err := c.Fetch() + if err != nil { + return nil, err + } + + data, err := c.Parse(res) + if err != nil { + return nil, err + } + + list, err := c.Format(latitude, longitude, data) + if err != nil { + return nil, err + } + + return list, nil +} + +func (c *CEA_DASE) getTimestamp(data string) int64 { + t, _ := time.Parse("02/01/2006 15:04:05", data) + return t.Add(-1 * time.Hour).UnixMilli() +} + +func (c *CEA_DASE) getMagnitude(data string) float64 { + m := strings.Split(data, "=") + if len(m) > 1 { + magnitude, _ := strconv.ParseFloat(m[1], 64) + return magnitude + } + + return 0 +} + +func (c *CEA_DASE) getLatitude(data string) float64 { + pos := strings.Split(data, ",") + if len(pos) > 1 { + latitude, _ := strconv.ParseFloat(pos[0], 64) + return latitude + } + + return 0 +} + +func (c *CEA_DASE) getLongitude(data string) float64 { + pos := strings.Split(data, ",") + if len(pos) > 1 { + longitude, _ := strconv.ParseFloat(pos[1], 64) + return longitude + } + + return 0 +} diff --git a/app/v1/trace/ingv.go b/app/v1/trace/ingv.go new file mode 100644 index 00000000..5bb994b0 --- /dev/null +++ b/app/v1/trace/ingv.go @@ -0,0 +1,157 @@ +package trace + +import ( + "encoding/csv" + "strconv" + "strings" + "time" + + "github.com/anyshake/observer/utils/duration" + "github.com/anyshake/observer/utils/request" +) + +type INGV struct { + DataSourceCache +} + +func (c *INGV) Property() string { + return "Istituto nazionale di geofisica e vulcanologia" +} + +func (c *INGV) Fetch() ([]byte, error) { + if duration.Difference(time.Now(), c.Time) <= EXPIRATION { + return c.Cache, nil + } + + res, err := request.GET( + "https://webservices.ingv.it/fdsnws/event/1/query?minmag=-1&format=text&timezone=UTC", + 10*time.Second, time.Second, 3, false, nil, + ) + if err != nil { + return nil, err + } + + c.Time = time.Now() + c.Cache = make([]byte, len(res)) + copy(c.Cache, res) + + return res, nil +} + +func (c *INGV) Parse(data []byte) (map[string]any, error) { + result := make(map[string]any) + result["data"] = make([]any, 0) + + csvDataStr := strings.ReplaceAll(string(data), "|", ",") + reader := csv.NewReader(strings.NewReader(csvDataStr)) + records, err := reader.ReadAll() + if err != nil { + return nil, err + } + + for _, record := range records[1:] { + item := make(map[string]any) + for i, v := range record { + switch i { + case 1: + item["timestamp"] = c.getTimestamp(v) + case 2: + item["latitude"] = c.getLatitude(v) + case 3: + item["longitude"] = c.getLongitude(v) + case 4: + item["depth"] = c.getDepth(v) + case 10: + item["magnitude"] = c.getMagnitude(v) + case 12: + item["event"] = v + item["region"] = v + } + } + result["data"] = append(result["data"].([]any), item) + } + + return result, nil +} + +func (c *INGV) Format(latitude, longitude float64, data map[string]any) ([]Event, error) { + var list []Event + for _, v := range data["data"].([]any) { + l := Event{ + Verfied: true, + Latitude: v.(map[string]any)["latitude"].(float64), + Longitude: v.(map[string]any)["longitude"].(float64), + Depth: v.(map[string]any)["depth"].(float64), + Event: v.(map[string]any)["event"].(string), + Region: v.(map[string]any)["region"].(string), + Timestamp: v.(map[string]any)["timestamp"].(int64), + Magnitude: v.(map[string]any)["magnitude"].(float64), + } + l.Distance = getDistance(latitude, l.Latitude, longitude, l.Longitude) + l.Estimation = getEstimation(l.Depth, l.Distance) + + list = append(list, l) + } + + return list, nil +} + +func (c *INGV) List(latitude, longitude float64) ([]Event, error) { + res, err := c.Fetch() + if err != nil { + return nil, err + } + + data, err := c.Parse(res) + if err != nil { + return nil, err + } + + list, err := c.Format(latitude, longitude, data) + if err != nil { + return nil, err + } + + return list, nil +} + +func (c *INGV) getTimestamp(data string) int64 { + t, _ := time.Parse("2006-01-02T15:04:05.000000", data) + return t.UnixMilli() +} + +func (c *INGV) getMagnitude(data string) float64 { + m, err := strconv.ParseFloat(data, 64) + if err == nil { + return m + } + + return 0 +} + +func (c *INGV) getDepth(data string) float64 { + d, err := strconv.ParseFloat(data, 64) + if err == nil { + return d + } + + return 0 +} + +func (c *INGV) getLatitude(data string) float64 { + lat, err := strconv.ParseFloat(data, 64) + if err == nil { + return lat + } + + return 0 +} + +func (c *INGV) getLongitude(data string) float64 { + lng, err := strconv.ParseFloat(data, 64) + if err == nil { + return lng + } + + return 0 +} diff --git a/app/v1/trace/module.go b/app/v1/trace/module.go index db4f3070..c7cfc910 100644 --- a/app/v1/trace/module.go +++ b/app/v1/trace/module.go @@ -19,14 +19,16 @@ import ( // @Success 200 {object} response.HttpResponse{data=[]Event} "Successfully read the list of earthquake events" func (t *Trace) RegisterModule(rg *gin.RouterGroup, options *app.ServerOptions) { sources := map[string]DataSource{ - "CWA": &CWA{}, - "HKO": &HKO{}, - "JMA": &JMA{}, - "KMA": &KMA{}, - "CEIC": &CEIC{}, - "USGS": &USGS{}, - "SCEA_E": &SCEA_E{}, - "SCEA_B": &SCEA_B{}, + "CWA": &CWA{}, + "HKO": &HKO{}, + "JMA": &JMA{}, + "KMA": &KMA{}, + "CEIC": &CEIC{}, + "USGS": &USGS{}, + "INGV": &INGV{}, + "SCEA_E": &SCEA_E{}, + "SCEA_B": &SCEA_B{}, + "CEA_DASE": &CEA_DASE{}, } rg.POST("/trace", func(c *gin.Context) { diff --git a/driver/dao/insert.go b/driver/dao/insert.go new file mode 100644 index 00000000..d698850b --- /dev/null +++ b/driver/dao/insert.go @@ -0,0 +1,10 @@ +package dao + +import ( + "github.com/anyshake/observer/publisher" + "gorm.io/gorm" +) + +func Insert(db *gorm.DB, gp *publisher.Geophone) error { + return db.Table(DB_TABLENAME).Create(&dbRecord{Geophone: *gp}).Error +} diff --git a/driver/dao/migrate.go b/driver/dao/migrate.go new file mode 100644 index 00000000..32d57a0a --- /dev/null +++ b/driver/dao/migrate.go @@ -0,0 +1,7 @@ +package dao + +import "gorm.io/gorm" + +func Migrate(db *gorm.DB) error { + return db.Table(DB_TABLENAME).AutoMigrate(&dbRecord{}) +} diff --git a/driver/dao/operation.go b/driver/dao/query.go similarity index 68% rename from driver/dao/operation.go rename to driver/dao/query.go index 4fdc7eac..1aca352f 100644 --- a/driver/dao/operation.go +++ b/driver/dao/query.go @@ -5,14 +5,6 @@ import ( "gorm.io/gorm" ) -func Migrate(db *gorm.DB) error { - return db.Table(DB_TABLENAME).AutoMigrate(&dbRecord{}) -} - -func Insert(db *gorm.DB, gp *publisher.Geophone) error { - return db.Table(DB_TABLENAME).Create(&dbRecord{Geophone: *gp}).Error -} - func Query(db *gorm.DB, start, end int64) ([]publisher.Geophone, error) { var records []dbRecord err := db.Table(DB_TABLENAME).Select("ts, ehz, ehe, ehn").Where("ts >= ? AND ts <= ?", start, end).Scan(&records).Error diff --git a/driver/seedlink/end.go b/driver/seedlink/end.go index 67047fbb..fc1bfc98 100644 --- a/driver/seedlink/end.go +++ b/driver/seedlink/end.go @@ -2,8 +2,6 @@ package seedlink import ( "net" - "strconv" - "strings" "time" "github.com/anyshake/observer/feature" @@ -51,16 +49,12 @@ func (*END) Callback(sl *SeedLinkGlobal, client *SeedLinkClient, options *featur bufTime = time.UnixMilli(timestamp).UTC() ) if bufTime.After(client.StartTime.UTC()) && bufTime.Before(client.EndTime.UTC()) { - var countData []int32 - for _, v := range strings.Split(data, "|") { - intData, err := strconv.Atoi(v) - if err != nil { - return err - } - countData = append(countData, int32(intData)) + countDataArr, err := publisher.DecodeInt32Array(data) + if err != nil { + return err } - err := SendSLPacket(conn, client, SeedLinkPacket{ - Channel: channel, Timestamp: timestamp, Count: countData, + err = SendSLPacket(conn, client, SeedLinkPacket{ + Channel: channel, Timestamp: timestamp, Count: countDataArr, }) if err != nil { return err diff --git a/frontend/dist/asset-manifest.json b/frontend/dist/asset-manifest.json index a294ed90..ae48be7e 100644 --- a/frontend/dist/asset-manifest.json +++ b/frontend/dist/asset-manifest.json @@ -1,7 +1,7 @@ { "files": { "main.css": "./static/css/main.7a040865.css", - "main.js": "./static/js/main.6c81bc75.js", + "main.js": "./static/js/main.6f5b5049.js", "static/css/594.d6bfd15f.chunk.css": "./static/css/594.d6bfd15f.chunk.css", "static/js/594.ef2a52fb.chunk.js": "./static/js/594.ef2a52fb.chunk.js", "static/js/846.88ce4ddb.chunk.js": "./static/js/846.88ce4ddb.chunk.js", @@ -49,6 +49,6 @@ }, "entrypoints": [ "static/css/main.7a040865.css", - "static/js/main.6c81bc75.js" + "static/js/main.6f5b5049.js" ] } \ No newline at end of file diff --git a/frontend/dist/index.html b/frontend/dist/index.html index 48610911..1ce05cbd 100644 --- a/frontend/dist/index.html +++ b/frontend/dist/index.html @@ -1 +1 @@ -