From d8bdb048c4223d38df0c1d551d52a4f87b49a80f Mon Sep 17 00:00:00 2001 From: Aadhavan Srinivasan Date: Mon, 3 Mar 2025 16:16:53 -0500 Subject: [PATCH] Added sqlite DB to cache translations; retrieve translations from DB if they're cached, otherwise fetch from API and cache results --- golangServer/go.mod | 2 + golangServer/go.sum | 8 +++ golangServer/testing.go | 100 +++++++++++++++++++++++++++++++---- golangServer/translations.db | Bin 0 -> 12288 bytes 4 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 golangServer/translations.db diff --git a/golangServer/go.mod b/golangServer/go.mod index cba44c7..3282ed2 100644 --- a/golangServer/go.mod +++ b/golangServer/go.mod @@ -15,6 +15,8 @@ require ( github.com/google/s2a-go v0.1.8 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.14.0 // indirect + github.com/jmoiron/sqlx v1.4.0 // indirect + github.com/mattn/go-sqlite3 v1.14.24 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect go.opentelemetry.io/otel v1.29.0 // indirect diff --git a/golangServer/go.sum b/golangServer/go.sum index c1ca9b7..646c74d 100644 --- a/golangServer/go.sum +++ b/golangServer/go.sum @@ -10,6 +10,7 @@ cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0 cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI= cloud.google.com/go/translate v1.12.3 h1:XJ7LipYJi80BCgVk2lx1fwc7DIYM6oV2qx1G4IAGQ5w= cloud.google.com/go/translate v1.12.3/go.mod h1:qINOVpgmgBnY4YTFHdfVO4nLrSBlpvlIyosqpGEgyEg= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -17,12 +18,19 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go/v2 v2.14.0 h1:f+jMrjBPl+DL9nI4IQzLUxMq7XrAqFYB7hBPqMNIe8o= github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= diff --git a/golangServer/testing.go b/golangServer/testing.go index 0e75d7f..c98f35e 100644 --- a/golangServer/testing.go +++ b/golangServer/testing.go @@ -2,17 +2,49 @@ package main import ( "context" + "database/sql" "encoding/json" "fmt" "log" "net/http" + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + translate "cloud.google.com/go/translate/apiv3" "cloud.google.com/go/translate/apiv3/translatepb" ) const project_id string = "india-translate-testing-452100" +type translationStruct struct { + En string `db:"english" json:"en"` + Hi string `db:"hindi" json:"hi"` + Bn string `db:"bengali" json:"bn"` + Mr string `db:"marathi" json:"mr"` + Ta string `db:"tamil" json:"ta"` + Te string `db:"telugu" json:"te"` + Ml string `db:"malayalam" json:"ml"` + Kn string `db:"kannada" json:"kn"` + Gu string `db:"gujarati" json:"gu"` + Or string `db:"oriya" json:"or"` + Ur string `db:"urdu" json:"ur"` + Lus string `db:"mizo" json:"lus"` + As string `db:"assamese" json:"as"` + Pa string `db:"punjabi" json:"pa"` + Mai string `db:"maithili" json:"nai"` + Mwr string `db:"marwari" json:"mwr"` + Sat string `db:"santali" json:"sat"` + Ne string `db:"nepali" json:"ne"` + Gom string `db:"konkani" json:"gom"` + Tcy string `db:"tulu" json:"tcy"` + Bho string `db:"bhojpuri" json:"bho"` + Doi string `db:"dogri" json:"doi"` + Mni_mtei string `db:"manipuri" json:"mni_mtei"` + Sd string `db:"sindhi" json:"sd"` + Awa string `db:"awadhi" json:"awa"` +} + var lang_codes []string = []string{ "hi", // Hindi "bn", // Bengali @@ -40,6 +72,39 @@ var lang_codes []string = []string{ "awa", // Awadhi } +var db *sqlx.DB + +/* + Returns the cached translation from the database, for the given english text. The first parameter + +indicates whether or not the translation exists. +*/ +func getCachedTranslation(data string) (bool, translationStruct) { + prepared, err := db.Preparex("SELECT * from TRANSLATIONS WHERE english = ? COLLATE NOCASE") + if err != nil { + panic(err) + } + + translations := translationStruct{} + err = prepared.Get(&translations, data) + if err != nil { + if err == sql.ErrNoRows { + return false, translations + } else { + panic(err) + } + } else { + return true, translations + } +} + +func addToDatabase(translation translationStruct) { + _, err := db.NamedExec(`INSERT INTO translations VALUES (:english, :hindi, :bengali, :marathi, :tamil, :telugu, :kannada, :malayalam, :oriya, :gujarati, :marwari, :urdu, :mizo, :assamese, :punjabi, :maithili, :santali, :nepali, :konkani, :tulu, :bhojpuri, :dogri, :manipuri, :sindhi, :awadhi)`, &translation) + if err != nil { + panic(err) + } +} + func translateText(text string, targetLang string) (result string, err error) { return translateTextHelper(project_id, "en-US", targetLang, text) } @@ -80,22 +145,39 @@ func handler(w http.ResponseWriter, r *http.Request) { queries := r.URL.Query() toTranslate := queries["query"][0] - langToTranslation := make(map[string]string) - for _, lang_code := range lang_codes { - translation, err := translateText(toTranslate, lang_code) + if ok, translation := getCachedTranslation(toTranslate); ok { + translationJson, _ := json.Marshal(translation) + fmt.Fprintf(w, "%v", string(translationJson)) + } else { + langToTranslation := make(map[string]string) + for _, lang_code := range lang_codes { + translation, err := translateText(toTranslate, lang_code) + if err != nil { + panic(err) + } + langToTranslation[lang_code] = translation + } + langToTranslationJson, _ := json.Marshal(langToTranslation) + translation := translationStruct{} + err := json.Unmarshal(langToTranslationJson, &translation) + translation.En = toTranslate // langToTranslation doesn't contain the english value + addToDatabase(translation) if err != nil { panic(err) } - langToTranslation[lang_code] = translation + fmt.Fprintf(w, "%v", string(langToTranslationJson)) } - - langToTranslationJson, _ := json.Marshal(langToTranslation) - fmt.Fprintf(w, "%v", string(langToTranslationJson)) - - // fmt.Fprintf(w, "Hi there, I love %s!", r.URL.Path[1:]) } func main() { + fmt.Printf("Starting server...") + var err error + db, err = sqlx.Connect("sqlite3", "translations.db") + if err != nil { + panic(err) + } + defer db.Close() + http.HandleFunc("/", handler) log.Fatal(http.ListenAndServe(":9090", nil)) } diff --git a/golangServer/translations.db b/golangServer/translations.db new file mode 100644 index 0000000000000000000000000000000000000000..c050a9dece5c996f2847714898120474d0b518fb GIT binary patch literal 12288 zcmeI$&u`jh7zc1erlmt7OzI(8wJ6@f+9s+=Xa`P{mQ^`aZKW;CVLe4J3InIcP%!G! zT}m68NE?S8P_<-I7sUZXsAC+E$TX3EBczuv{Rev7j(cq3l*4+SG(8BvejmTTp7&#X zz~6pzTQ$=1_xWN@81m=R`%)+*eJRV5B+b$>Lr3rlQ{|l?pmS)d{6l&OB+R@mo#Ch zdA&4U58qf>URqt2SC_uJy(~|+L_Ue6^^B&LvhwQkcdPQ(D>v^ft=yGwE#JK!$*TIg z>R-PeNl`bU1yN2Eg^>*+L*!I#($7e1q z>y&0N&M0faQmJfyqfov~Vm+U^tmkOJR8xtZE-}Od5d@zm!|%@DQbH27r++Bj*XY0e znkXB2KVF-@4{M3($opS^#rtJN=?F(ZmgY)S@P`FhfCX591z3OuSbzmsfCX6K@y+nH zt8pcsOvb-hj4KO@5|1j%;-aENKT4$Xn=oCN`!N56S%*19V-L-3G+eJVZ5&qo4$r(M zdl&W@>@JQgu=}uEIIh8|!s(z(l@6RCoMSj=a8GekfxGXQ&(M2{{vYVo(XXKQ9Q~SC z_Ri30<3$CXD%yKE`U&ki+KyK-_i#w1L*;1e;+Kooc{8ULgsh8lLFgIE?$x7T(R9!l z!m`n-pyj~Yhcz5`e;;>u$6a^aeeP}OkTyY;A) z&5P~P&PDsYJEAs{abTYLUk~R=By?>)kw_*NlS(p?T=*b)CmK&EN+KCuP_k)F%fqxh zHFJPNTEX)?=lOP51H9X&Ddn0CwForP8*2+*3F%x|FpA*M>Xr5_jux zhj1PAPSJDG-$rkUj)jhm&HyiJ=+x1nD-PNg+BVt)9M#Y^aYPvZvh>T!%b!PEqn#UB zH6>IuEHrF127cCcFKdFf2Cz)oJGE-C9JC%$P=!?|b6BLk@3S|EecYurrkC**%u^gz zgIiFA`xtHmCzMdzmm=l^{}zzl^w^2c`N?SOyg8x?Xc`*i-N^_&+u_)2DfT+{7eD4K qzyd750xZA+EWiRRzyd750xZA+{|kZHnX4gsT@}7EGyk6ds^uU1NRaCQ literal 0 HcmV?d00001