internal/postgres: store excluded prefixes in memory
Read the excluded_prefixes table into memory, refreshing it
periodically.
Updates b/154958188.
CI-Result: Cloud Build
@@ -7,6 +7,9 @@ package postgres
import (
"context"
"database/sql"
+ "strings"
+ "sync"
+ "time"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/log"
@@ -16,20 +19,20 @@ import (
func (db *DB) IsExcluded(ctx context.Context, path string) (_ bool, err error) {
defer derrors.Wrap(&err, "DB.IsExcluded(ctx, %q)", path)
- const query = "SELECT prefix FROM excluded_prefixes WHERE starts_with($1, prefix);"
- row := db.db.QueryRow(ctx, query, path)
- var prefix string
- err = row.Scan(&prefix)
- switch err {
- case nil:
+ db.ensureExcludedPrefixes(ctx)
+ excludedPrefixes.mu.Lock()
+ defer excludedPrefixes.mu.Unlock()
+ if excludedPrefixes.err != nil {
+ return false, excludedPrefixes.err
+ }
+ for _, prefix := range excludedPrefixes.prefixes {
+ if strings.HasPrefix(path, prefix) {
log.Infof(ctx, "path %q matched excluded prefix %q", path, prefix)
return true, nil
- case sql.ErrNoRows:
- return false, nil
- default:
- return false, err
}
}
+ return false, nil
+}
// InsertExcludedPrefix inserts prefix into the excluded_prefixes table.
//
@@ -41,5 +44,57 @@ func (db *DB) InsertExcludedPrefix(ctx context.Context, prefix, user, reason str
_, err = db.db.Exec(ctx, "INSERT INTO excluded_prefixes (prefix, created_by, reason) VALUES ($1, $2, $3)",
prefix, user, reason)
+ if err != nil {
+ // Arrange to re-read the excluded_prefixes table on the next call to IsExcluded.
+ excludedPrefixes.mu.Lock()
+ excludedPrefixes.lastFetched = time.Time{}
+ excludedPrefixes.mu.Unlock()
+ }
return err
}
+
+// In-memory copy of excluded_prefixes.
+var excludedPrefixes struct {
+ mu sync.Mutex
+ prefixes []string
+ err error
+ lastFetched time.Time
+}
+
+const excludedPrefixesExpiration = time.Minute
+
+// refreshExcludedPrefixes makes sure the in-memory copy of the
+// excluded_prefixes table is up to date.
+func (db *DB) ensureExcludedPrefixes(ctx context.Context) {
+ excludedPrefixes.mu.Lock()
+ lastFetched := excludedPrefixes.lastFetched
+ excludedPrefixes.mu.Unlock()
+ if time.Since(lastFetched) < excludedPrefixesExpiration {
+ return
+ }
+ prefixes, err := db.readExcludedPrefixes(ctx)
+ excludedPrefixes.mu.Lock()
+ defer excludedPrefixes.mu.Unlock()
+ excludedPrefixes.prefixes = prefixes
+ excludedPrefixes.err = err
+ if err != nil {
+ log.Errorf(ctx, "reading excluded_prefixes: %v", err)
+ }
+}
+
+// readExcludedPrefixes reads all the excluded prefixes from the database.
+func (db *DB) readExcludedPrefixes(ctx context.Context) ([]string, error) {
+ var eps []string
+ err := db.db.RunQuery(ctx, `SELECT prefix FROM excluded_prefixes`, func(rows *sql.Rows) error {
+ var ep string
+ if err := rows.Scan(&ep); err != nil {
+ return err
+ }
+ eps = append(eps, ep)
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ return eps, nil
+}
