Skip to content

ElasticSearch fixes #195

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Swan merged 2 commits into from
Oct 5, 2026
Merged

ElasticSearch fixes #195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6a53dc5
remove unused es index
AiAe Oct 4, 2026
f4b7515
Fix ES indexing issues on insert/updating mapset
AiAe Oct 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion db/elasticsearch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (

var ElasticSearch *elasticsearch.Client

const elasticMapsetIndex = "mapsets-v2"
const elasticMapSearchIndex = "maps"

// InitializeElasticSearch Initializes the ElasticSearch client
Expand Down
35 changes: 31 additions & 4 deletions db/elasticsearch_mapsets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ func IndexElasticSearchMapset(mapset Mapset) error {
for _, mapQua := range mapset.Maps {
elasticMap := ElasticMap{
MapQua: mapQua,
PackageMD5: mapset.PackageMD5,
DateSubmitted: mapset.DateSubmitted,
DateLastUpdated: mapset.DateLastUpdated,
Explicit: mapset.IsExplicit,
Expand All @@ -210,13 +211,13 @@ func IndexElasticSearchMapset(mapset Mapset) error {
}

resp, err := ElasticSearch.Create(elasticMapSearchIndex,
fmt.Sprintf("%v", mapQua.Id), bytes.NewReader(data))
strconv.Itoa(mapQua.Id), bytes.NewReader(data))

if err != nil {
return err
}

defer resp.Body.Close()
resp.Body.Close()
}

return nil
Expand All @@ -229,19 +230,23 @@ func UpdateElasticSearchMapset(mapset Mapset) error {
for _, mapQua := range mapset.Maps {
elasticMap := ElasticMap{
MapQua: mapQua,
PackageMD5: mapset.PackageMD5,
DateSubmitted: mapset.DateSubmitted,
DateLastUpdated: mapset.DateLastUpdated,
Explicit: mapset.IsExplicit,
MapsetPlayCount: mapsetPlayCount,
}

data, err := json.Marshal(&elasticMap)
data, err := json.Marshal(struct {
Doc ElasticMap `json:"doc"`
DocAsUpsert bool `json:"doc_as_upsert"`
}{Doc: elasticMap, DocAsUpsert: true})

if err != nil {
return err
}

resp, err := ElasticSearch.Update(elasticMapsetIndex, fmt.Sprintf("%v", mapQua.Id), bytes.NewReader(data))
resp, err := ElasticSearch.Update(elasticMapSearchIndex, strconv.Itoa(mapQua.Id), bytes.NewReader(data))

if err != nil {
return err
Expand All @@ -253,6 +258,28 @@ func UpdateElasticSearchMapset(mapset Mapset) error {
return nil
}

// UpdateElasticSearchMapDifficulty Updates only the rating of the matching uploaded map version.
func UpdateElasticSearchMapDifficulty(id int, md5 string, difficultyRating float64) error {
data, err := json.Marshal(map[string]interface{}{
"script": map[string]interface{}{
"source": "if (ctx._source.md5 == params.md5) { ctx._source.difficulty_rating = params.difficulty_rating } else { ctx.op = 'noop' }",
"params": map[string]interface{}{
"md5": md5,
"difficulty_rating": difficultyRating,
},
},
})
if err != nil {
return err
}
resp, err := ElasticSearch.Update(elasticMapSearchIndex, strconv.Itoa(id), bytes.NewReader(data), ElasticSearch.Update.WithRetryOnConflict(3))
if err != nil {
return err
}
resp.Body.Close()
return nil
}

// DeleteElasticSearchMapset Deletes an individual mapset in elastic
func DeleteElasticSearchMapset(id int) error {
queryMap := map[string]interface{}{
Expand Down
31 changes: 20 additions & 11 deletions db/maps.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,7 @@ func (m *MapQua) TableName() string {
}

func (m *MapQua) AfterFind(*gorm.DB) error {
m.MaxCombo = m.CountHitObjectLong*2 + m.CountHitObjectNormal
m.PlayAttempts = m.PlayCount + m.FailCount

if m.CountHitObjectLong != 0 {
m.LongNotePercentage = float32(m.CountHitObjectLong) /
float32(m.CountHitObjectNormal+m.CountHitObjectLong) * 100
}
m.UpdateComputedValues()

if m.DateClanRanked != nil {
t := time.UnixMilli(*m.DateClanRanked)
Expand All @@ -65,6 +59,18 @@ func (m *MapQua) AfterFind(*gorm.DB) error {
return nil
}

// UpdateComputedValues Calculates display and search fields from the map's stored counts.
func (m *MapQua) UpdateComputedValues() {
m.MaxCombo = m.CountHitObjectLong*2 + m.CountHitObjectNormal
m.PlayAttempts = m.PlayCount + m.FailCount

m.LongNotePercentage = 0
if m.CountHitObjectNormal+m.CountHitObjectLong > 0 {
m.LongNotePercentage = float32(m.CountHitObjectLong) /
float32(m.CountHitObjectNormal+m.CountHitObjectLong) * 100
}
}

func (m *MapQua) String() string {
return fmt.Sprintf("%v - %v [%v]", m.Artist, m.Title, m.DifficultyName)
}
Expand Down Expand Up @@ -151,13 +157,16 @@ func DeleteMap(id int) error {
return result.Error
}

// UpdateMapDifficultyRating Updates the difficulty rating of a map
func UpdateMapDifficultyRating(id int, difficultyRating float64) error {
// UpdateMapDifficultyRating Saves and indexes a calculation only for the matching uploaded map version.
func UpdateMapDifficultyRating(id int, md5 string, difficultyRating float64) error {
result := SQL.Model(&MapQua{}).
Where("id = ?", id).
Where("id = ? AND md5 = ?", id, md5).
Update("difficulty_rating", difficultyRating)

return result.Error
if result.Error != nil {
return result.Error
}
return UpdateElasticSearchMapDifficulty(id, md5, difficultyRating)
}

// UpdateMapClanRanked Updates the clan ranked status of a map
Expand Down
27 changes: 20 additions & 7 deletions db/mapsets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,24 +186,31 @@ func GetMapsetOnlineOffsets() (interface{}, error) {
return offsets, nil
}

// RankMapset Ranks all maps in a mapset
func RankMapset(id int) error {
// RankMapset Ranks all maps in SQL and synchronizes the loaded mapset for indexing.
func RankMapset(mapset *Mapset) error {
result := SQL.Model(&MapQua{}).
Where("mapset_id = ?", id).
Where("mapset_id = ?", mapset.Id).
Update("ranked_status", enums.RankedStatusRanked)

if result.Error != nil {
return result.Error
}

dateLastUpdated := time.Now().UnixMilli()
result = SQL.Model(&Mapset{}).
Where("id = ?", id).
Update("date_last_updated", time.Now().UnixMilli())
Where("id = ?", mapset.Id).
Update("date_last_updated", dateLastUpdated)

if result.Error != nil {
return result.Error
}

for _, songMap := range mapset.Maps {
songMap.RankedStatus = enums.RankedStatusRanked
}
mapset.DateLastUpdated = dateLastUpdated
mapset.DateLastUpdatedJSON = time.UnixMilli(dateLastUpdated)

return nil
}

Expand Down Expand Up @@ -249,6 +256,7 @@ func UpdateMapsetPackageMD5(id int, md5 string) error {

// UpdateMetadata Updates the metadata of a given mapset (username, artist, title, etc)
func (m *Mapset) UpdateMetadata() error {
dateLastUpdated := time.Now().UnixMilli()
result := SQL.Model(&Mapset{}).
Where("id = ?", m.Id).
Updates(map[string]interface{}{
Expand All @@ -257,10 +265,15 @@ func (m *Mapset) UpdateMetadata() error {
"title": m.Title,
"source": m.Source,
"tags": m.Tags,
"date_last_updated": time.Now().UnixMilli(),
"date_last_updated": dateLastUpdated,
})

return result.Error
if result.Error != nil {
return result.Error
}
m.DateLastUpdated = dateLastUpdated
m.DateLastUpdatedJSON = time.UnixMilli(dateLastUpdated)
return nil
}

// UpdateExplicit Sets the explicit state of the mapset
Expand Down
63 changes: 40 additions & 23 deletions handlers/mapset_submission.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
"github.com/Quaver/api2/qua"
"github.com/Quaver/api2/sliceutil"
"github.com/Quaver/api2/tools"
v1 "github.com/Quaver/api2/v1"
"github.com/Quaver/api2/webhooks"
"github.com/gabriel-vasile/mimetype"
"github.com/gin-gonic/gin"
Expand Down Expand Up @@ -53,6 +52,8 @@ const (
errAudioPreviewFileNoExists string = "could not create audio preview (file no exists)"
)

type mapDifficultyFiles map[*db.MapQua]string

// HandleMapsetSubmission Handles the uploading/updating of a mapset archive (.qp) file
// Endpoint: POST /v2/mapset
func HandleMapsetSubmission(c *gin.Context) *APIError {
Expand All @@ -62,6 +63,15 @@ func HandleMapsetSubmission(c *gin.Context) *APIError {
return nil
}

difficultyFiles := make(mapDifficultyFiles)
defer func() {
for _, filePath := range difficultyFiles {
if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) {
logrus.Error("Error removing map difficulty file: ", filePath, err)
}
}
}()

zipReader, apiErr := checkValidRequestMapset(c, user)

if apiErr != nil {
Expand Down Expand Up @@ -94,9 +104,9 @@ func HandleMapsetSubmission(c *gin.Context) *APIError {
var mapset *db.Mapset

if isUploadingNewMapset {
mapset, apiErr = uploadNewMapset(user, quaFiles)
mapset, apiErr = uploadNewMapset(user, quaFiles, difficultyFiles)
} else {
mapset, apiErr = updateExistingMapset(user, quaFiles)
mapset, apiErr = updateExistingMapset(user, quaFiles, difficultyFiles)
}

if apiErr != nil {
Expand All @@ -123,8 +133,9 @@ func HandleMapsetSubmission(c *gin.Context) *APIError {
return APIErrorServerError("Error updating elastic search", err)
}

if err := v1.UpdateElasticSearchMapset(mapset.Id); err != nil {
logrus.Error(err)
for songMap, filePath := range difficultyFiles {
delete(difficultyFiles, songMap)
go calcMapDifficulty(songMap.Id, songMap.MD5, filePath)
}

if apiErr := resolveMapsetInRankingQueue(user, mapset); apiErr != nil {
Expand Down Expand Up @@ -314,7 +325,7 @@ func validateQuaFiles(user *db.User, quaFiles map[*zip.File]*qua.Qua) *APIError
}

// Handles the uploading of a brand new mapset
func uploadNewMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua) (*db.Mapset, *APIError) {
func uploadNewMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua, difficultyFiles mapDifficultyFiles) (*db.Mapset, *APIError) {
if apiErr := checkUserUploadEligibility(user); apiErr != nil {
return nil, apiErr
}
Expand All @@ -335,7 +346,7 @@ func uploadNewMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua) (*db.Mapset
}

for _, quaFile := range quaFiles {
songMap, apiErr := InsertOrUpdateMap(user, mapset, quaFile)
songMap, apiErr := InsertOrUpdateMap(user, mapset, quaFile, difficultyFiles)

if apiErr != nil {
return nil, apiErr
Expand All @@ -356,7 +367,7 @@ func uploadNewMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua) (*db.Mapset
}

// Handles the updating of an existing mapset
func updateExistingMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua) (*db.Mapset, *APIError) {
func updateExistingMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua, difficultyFiles mapDifficultyFiles) (*db.Mapset, *APIError) {
quaSlice := sliceutil.Values(quaFiles)

if !sliceutil.All(quaSlice, func(q *qua.Qua) bool {
Expand Down Expand Up @@ -400,7 +411,7 @@ func updateExistingMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua) (*db.M

// Insert new maps & update existing ones
for _, quaFile := range quaFiles {
songMap, apiErr := InsertOrUpdateMap(user, mapset, quaFile)
songMap, apiErr := InsertOrUpdateMap(user, mapset, quaFile, difficultyFiles)

if apiErr != nil {
return nil, apiErr
Expand Down Expand Up @@ -440,7 +451,7 @@ func updateExistingMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua) (*db.M
}

// InsertOrUpdateMap Inserts/Updates a map in the database
func InsertOrUpdateMap(user *db.User, mapset *db.Mapset, quaFile *qua.Qua) (*db.MapQua, *APIError) {
func InsertOrUpdateMap(user *db.User, mapset *db.Mapset, quaFile *qua.Qua, difficultyFiles mapDifficultyFiles) (*db.MapQua, *APIError) {
songMap := &db.MapQua{
MapsetId: mapset.Id,
CreatorId: user.Id,
Expand All @@ -457,8 +468,8 @@ func InsertOrUpdateMap(user *db.User, mapset *db.Mapset, quaFile *qua.Qua) (*db.
BPM: quaFile.CommonBPM(),
CountHitObjectNormal: quaFile.CountHitObjectNormal(),
CountHitObjectLong: quaFile.CountHitObjectLong(),
MaxCombo: quaFile.MaxCombo(),
}
songMap.UpdateComputedValues()

if quaFile.MapId != -1 {
songMap.Id = quaFile.MapId
Expand All @@ -475,7 +486,15 @@ func InsertOrUpdateMap(user *db.User, mapset *db.Mapset, quaFile *qua.Qua) (*db.
return nil, APIErrorServerError("Error saving map in db", err)
}

filePath := fmt.Sprintf("%v/%v.qua", files.GetTempDirectory(), songMap.Id)
tempMapFile, err := os.CreateTemp(files.GetTempDirectory(), fmt.Sprintf("%d-*.qua", songMap.Id))
if err != nil {
return nil, APIErrorServerError("Error creating temporary map file", err)
}
filePath := tempMapFile.Name()
difficultyFiles[songMap] = filePath
if err := tempMapFile.Close(); err != nil {
return nil, APIErrorServerError("Error closing temporary map file", err)
}

if err := quaFile.Write(filePath); err != nil {
return nil, APIErrorServerError("Error writing .qua file to disk", err)
Expand All @@ -485,14 +504,6 @@ func InsertOrUpdateMap(user *db.User, mapset *db.Mapset, quaFile *qua.Qua) (*db.
return nil, APIErrorServerError("Error uploading .qua file to azure", err)
}

go func() {
calcMapDifficulty(songMap, filePath)

if err := os.Remove(filePath); err != nil {
logrus.Error("Error removing file: ", filePath)
}
}()

return songMap, nil
}

Expand Down Expand Up @@ -560,16 +571,22 @@ func getUserMaxUploadsPerMonth(user *db.User) int {
}

// Calculates a map's difficulty rating
func calcMapDifficulty(songMap *db.MapQua, filePath string) {
func calcMapDifficulty(id int, md5 string, filePath string) {
defer func() {
if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) {
logrus.Error("Error removing map difficulty file: ", filePath, err)
}
}()

calc, err := tools.RunDifficultyCalculator(filePath, 0)

if err != nil {
logrus.Error("Error calculating difficulty for map: ", err)
return
}

if err := db.UpdateMapDifficultyRating(songMap.Id, calc.Difficulty.OverallDifficulty); err != nil {
logrus.Error("Error updating map difficulty rating in DB: ", err)
if err := db.UpdateMapDifficultyRating(id, md5, calc.Difficulty.OverallDifficulty); err != nil {
logrus.Errorf("Error saving difficulty rating for map %d: %v", id, err)
return
}
}
Expand Down
Loading
Loading