Rockingcool/india-translator
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added sqlite DB to cache translations; retrieve translations from DB if they're cached, otherwise fetch from API and cache results

Browse Source
This commit is contained in:
Aadhavan Srinivasan
2025-03-03 16:16:53 -05:00
parent a39fcc69a4
commit d8bdb048c4
4 changed files with 101 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
golangServer/go.mod
View File

@@ -15,6 +15,8 @@ require (
github.com/google/s2a-go v0.1.8 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
github.com/googleapis/gax-go/v2 v2.14.0 // indirect
github.com/jmoiron/sqlx v1.4.0 // indirect
github.com/mattn/go-sqlite3 v1.14.24 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
go.opentelemetry.io/otel v1.29.0 // indirect

8
golangServer/go.sum
View File

@@ -10,6 +10,7 @@ cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0
cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI=
cloud.google.com/go/translate v1.12.3 h1:XJ7LipYJi80BCgVk2lx1fwc7DIYM6oV2qx1G4IAGQ5w=
cloud.google.com/go/translate v1.12.3/go.mod h1:qINOVpgmgBnY4YTFHdfVO4nLrSBlpvlIyosqpGEgyEg=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -17,12 +18,19 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM=
github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA=
github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw=
github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=
github.com/googleapis/gax-go/v2 v2.14.0 h1:f+jMrjBPl+DL9nI4IQzLUxMq7XrAqFYB7hBPqMNIe8o=
github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk=
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=

100
golangServer/testing.go
View File

@@ -2,17 +2,49 @@ package main
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"log"
"net/http"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
translate "cloud.google.com/go/translate/apiv3"
"cloud.google.com/go/translate/apiv3/translatepb"
)
const project_id string = "india-translate-testing-452100"
type translationStruct struct {
En string `db:"english" json:"en"`
Hi string `db:"hindi" json:"hi"`
Bn string `db:"bengali" json:"bn"`
Mr string `db:"marathi" json:"mr"`
Ta string `db:"tamil" json:"ta"`
Te string `db:"telugu" json:"te"`
Ml string `db:"malayalam" json:"ml"`
Kn string `db:"kannada" json:"kn"`
Gu string `db:"gujarati" json:"gu"`
Or string `db:"oriya" json:"or"`
Ur string `db:"urdu" json:"ur"`
Lus string `db:"mizo" json:"lus"`
As string `db:"assamese" json:"as"`
Pa string `db:"punjabi" json:"pa"`
Mai string `db:"maithili" json:"nai"`
Mwr string `db:"marwari" json:"mwr"`
Sat string `db:"santali" json:"sat"`
Ne string `db:"nepali" json:"ne"`
Gom string `db:"konkani" json:"gom"`
Tcy string `db:"tulu" json:"tcy"`
Bho string `db:"bhojpuri" json:"bho"`
Doi string `db:"dogri" json:"doi"`
Mni_mtei string `db:"manipuri" json:"mni_mtei"`
Sd string `db:"sindhi" json:"sd"`
Awa string `db:"awadhi" json:"awa"`
}
var lang_codes []string = []string{
"hi", // Hindi
"bn", // Bengali
@@ -40,6 +72,39 @@ var lang_codes []string = []string{
"awa", // Awadhi
}
var db *sqlx.DB
/*
Returns the cached translation from the database, for the given english text. The first parameter
indicates whether or not the translation exists.
*/
func getCachedTranslation(data string) (bool, translationStruct) {
prepared, err := db.Preparex("SELECT * from TRANSLATIONS WHERE english = ? COLLATE NOCASE")
if err != nil {
panic(err)
}
translations := translationStruct{}
err = prepared.Get(&translations, data)
if err != nil {
if err == sql.ErrNoRows {
return false, translations
} else {
panic(err)
}
} else {
return true, translations
}
}
func addToDatabase(translation translationStruct) {
_, err := db.NamedExec(`INSERT INTO translations VALUES (:english, :hindi, :bengali, :marathi, :tamil, :telugu, :kannada, :malayalam, :oriya, :gujarati, :marwari, :urdu, :mizo, :assamese, :punjabi, :maithili, :santali, :nepali, :konkani, :tulu, :bhojpuri, :dogri, :manipuri, :sindhi, :awadhi)`, &translation)
if err != nil {
panic(err)
}
}
func translateText(text string, targetLang string) (result string, err error) {
return translateTextHelper(project_id, "en-US", targetLang, text)
}
@@ -80,22 +145,39 @@ func handler(w http.ResponseWriter, r *http.Request) {
queries := r.URL.Query()
toTranslate := queries["query"][0]
langToTranslation := make(map[string]string)
for _, lang_code := range lang_codes {
translation, err := translateText(toTranslate, lang_code)
if ok, translation := getCachedTranslation(toTranslate); ok {
translationJson, _ := json.Marshal(translation)
fmt.Fprintf(w, "%v", string(translationJson))
} else {
langToTranslation := make(map[string]string)
for _, lang_code := range lang_codes {
translation, err := translateText(toTranslate, lang_code)
if err != nil {
panic(err)
}
langToTranslation[lang_code] = translation
}
langToTranslationJson, _ := json.Marshal(langToTranslation)
translation := translationStruct{}
err := json.Unmarshal(langToTranslationJson, &translation)
translation.En = toTranslate // langToTranslation doesn't contain the english value
addToDatabase(translation)
if err != nil {
panic(err)
}
langToTranslation[lang_code] = translation
fmt.Fprintf(w, "%v", string(langToTranslationJson))
}
langToTranslationJson, _ := json.Marshal(langToTranslation)
fmt.Fprintf(w, "%v", string(langToTranslationJson))
// fmt.Fprintf(w, "Hi there, I love %s!", r.URL.Path[1:])
}
func main() {
fmt.Printf("Starting server...")
var err error
db, err = sqlx.Connect("sqlite3", "translations.db")
if err != nil {
panic(err)
}
defer db.Close()
http.HandleFunc("/", handler)
log.Fatal(http.ListenAndServe(":9090", nil))
}

BIN
golangServer/translations.db Normal file
View File

Binary file not shown.