Javascript, Caching to Optimize Network Traffic
Browsers use caching to temporarily store resources locally on the device, such as HTML, CSS, and JavaScript files, images, and videos. This allows the browser to quickly access the resources without having to make additional requests to the server.
One way browsers can enable device-side caching is through the use of local storage. Local storage is a key-value storage system that allows web applications to store data on the client-side. Data stored in local storage is persistent and does not expire, unlike cookies. The data is also accessible across different browser tabs and windows, as well as across browser sessions.
Here is an example of how to use local storage in JavaScript:
// Setting a value in local storage
localStorage.setItem("key", "value");
// Retrieving a value from local storage
let value = localStorage.getItem("key");
console.log(value); // Output: "value"
// Removing a value from local storage
localStorage.removeItem("key");Another way to store information on the client-side is through the use of IndexedDB. IndexedDB is a transactional, non-relational, object-oriented database system that allows web applications to store large amounts of data on the client-side. It is more powerful than local storage and is better suited for storing structured data, such as arrays and objects.
Here is an example of how to use IndexedDB in JavaScript:
// Open a database
let request = indexedDB.open("databaseName", 1);
// Create an object store
request.onupgradeneeded = function() {
let db = request.result;
let objectStore = db.createObjectStore("objectStoreName", {keyPath: "id"});
}
// Add data to the object store
let db = request.result;
let transaction = db.transaction("objectStoreName", "readwrite");
let objectStore = transaction.objectStore("objectStoreName");
let data = {id: 1, name: "John Doe"};
let request = objectStore.add(data);
// Retrieve data from the object store
let transaction = db.transaction("objectStoreName", "readonly");
let objectStore = transaction.objectStore("objectStoreName");
let request = objectStore.get(1);
request.onsuccess = function() {
let data = request.result;
console.log(data.name); // Output: "John Doe"
}Other means of caching that can be used include Application Cache, Cache API, and Service Workers.
Application Cache allows a web page to store resources locally so that they can be accessed offline. The resources are specified in a manifest file, which the browser checks for updates every time the page is loaded.
Cache API is a programmatic way to cache resources and is useful when the resources are dynamically generated or change frequently.
Service Workers are a way to intercept network requests and handle them programmatically, allowing the browser to control the caching strategy.
Here is an example of how you can check the MD5 checksum of a local file in JavaScript using the crypto-js library:
// Import the crypto-js library
const CryptoJS = require("crypto-js");
// Read the local file as a binary string
const file = new FileReader();
file.readAsBinaryString(localFile);
// Calculate the MD5 checksum of the file
const md5 = CryptoJS.MD5(file.result).toString();
console.log(md5);
In order to check the MD5 checksum of a file on AWS S3, you would need to use the AWS SDK for JavaScript in Node.js, since the AWS S3 service is a cloud object storage service provided by Amazon.
Here is an example of how you can use the AWS SDK for JavaScript in Node.js to check the MD5 checksum of a file on AWS S3:
// Import the AWS SDK for JavaScript in Node.js
const AWS = require('aws-sdk');
// Configure your AWS credentials
AWS.config.update({
accessKeyId: 'ACCESS_KEY_ID',
secretAccessKey: 'SECRET_ACCESS_KEY'
});
// Create an instance of the S3 client
const s3 = new AWS.S3();
// Define the S3 bucket and key of the file you want to check
const params = {
Bucket: 'my-bucket',
Key: 'path/to/file.jpg'
};
// Use the headObject method to retrieve the ETag of the file
s3.headObject(params, function(err, data) {
if (err) {
console.log(err);
} else {
// The ETag is the MD5 checksum of the file
console.log(data.ETag);
}
});You can compare the checksum obtained by the above code to the one you calculated for your local file.
Please note that for security reasons, it’s better to use the AWS IAM roles with the appropriate permissions for the s3 bucket rather than providing the access key and secret access key.
Retrieving OSM data with internal caching
Here is an example of using JavaScript caching to retrieve OpenStreetMap (OSM) map data for an area of interest and caching the results using local storage if that area of interest is previously queried:
// Define the area of interest
const bounds = {
left: -122.67,
bottom: 45.52,
right: -122.45,
top: 45.60
};
// Get the OSM map data for the area of interest
function getOSMMapData() {
// Check if the map data for the current area of interest is already in local storage
const mapData = localStorage.getItem(`osmMapData_${bounds.left}_${bounds.bottom}_${bounds.right}_${bounds.top}`);
if (mapData) {
// If the map data is in local storage, return it
return JSON.parse(mapData);
} else {
// If the map data is not in local storage, fetch it from the server
return fetch(`https://overpass-api.de/api/map?bbox=${bounds.left},${bounds.bottom},${bounds.right},${bounds.top}`)
.then(response => response.json())
.then(data => {
// Store the map data in local storage with the area of interest as key
localStorage.setItem(`osmMapData_${bounds.left}_${bounds.bottom}_${bounds.right}_${bounds.top}`, JSON.stringify(data));
return data;
});
}
}In this example, the map data is stored in local storage using the area of interest as the key, this way it can be easily retrieved when the same area of interest is queried again. This way the data is stored for the specific area of interest, So next time the same area of interest is requested, it can be fetched from the local storage rather than making a request to the server again. You can add MD5 sum check to verify the content before returning the cached data as well.
Retrieving S3 data with internal caching
Retrieving files from AWS S3 directly from the browser is possible with the use of signed URLs or pre-signed URLs. These URLs allow the browser to access the AWS S3 REST APIs directly without the need for additional authentication. To generate a signed URL, you would need to use the AWS SDK for JavaScript in Node.js on the server-side and create a server-side endpoint that generates the signed URL.
Here is an example of generating a signed URL using the AWS SDK for JavaScript in Node.js.
Server Side:
const AWS = require('aws-sdk');
// Configure your AWS credentials
AWS.config.update({
accessKeyId: 'ACCESS_KEY_ID',
secretAccessKey: 'SECRET_ACCESS_KEY',
region: 'REGION'
});
const s3 = new AWS.S3();
function getSignedUrl(bucket, key) {
// Set the parameters for the signed URL
const params = {
Bucket: bucket,
Key: key,
Expires: 3600
};
// Generate the signed URL
return s3.getSignedUrl('getObject', params);
}
app.get('/signed-url', (req, res) => {
const { bucket, key } = req.query;
const signedUrl = getSignedUrl(bucket, key);
res.json({ url: signedUrl });
});
app.get('/get-etag', (req, res) => {
const { bucket, key } = req.query;
const params = {
Bucket: bucket,
Key: key
};
// Use the headObject method to retrieve the ETag of the file
s3.headObject(params, function(err, data) {
if (err) {
res.send(err);
} else {
// The ETag is the MD5 checksum of the file
res.json({ etag: data.ETag });
}
});
});
Here is an example of how you can check the ETag of a file in AWS S3, compare it with the MD5 checksum of the file stored in IndexedDB, and retrieve the file from S3 if the ETag or MD5 checksum does not match, all in one function:
function checkEtagAndDownload(bucket, key) {
// Make a request to the server to get the ETag
return fetch(`/get-etag?bucket=${bucket}&key=${key}`)
.then(response => response.json())
.then(data => {
// Open the database
let request = indexedDB.open("awsS3Database", 1);
request.onsuccess = function() {
let db = request.result;
let transaction = db.transaction("awsS3Store", "readonly");
let objectStore = transaction.objectStore("awsS3Store");
let request = objectStore.get(key);
request.onsuccess = function() {
let s3data = request.result;
if (s3data) {
var md5 = CryptoJS.MD5(s3data.data).toString();
// Compare the ETag from the server with the MD5 checksum from the IndexedDB
if (md5 === s3data.md5 && md5 === data.etag) {
return s3data.data;
} else {
// Download the file from AWS S3 if the ETag or MD5 does not match
let url = getSignedUrl(bucket,key);
return fetch(url)
.then(response => response.blob())
.then(data => {
var md5 = CryptoJS.MD5(data).toString();
var dataObject = {data: data, md5: md5, etag: data.etag};
let transaction = db.transaction("awsS3Store", "readwrite");
let objectStore = transaction.objectStore("awsS3Store");
let request = objectStore.put(dataObject,key);
request.onsuccess = function () {
return data;
}
});
}
}
else{
// Download the file from AWS S3 if the data is not found in indexedDB
let url = getSignedUrl(bucket,key);
return fetch(url)
.then(response => response.blob())
.then(data => {
var md5 = CryptoJS.MD5(data).toString();
var dataObject = {data: data, md5: md5, etag: data.etag};
let transaction = db.transaction("awsS3Store", "readwrite");
let objectStore = transaction.objectStore("awsS3Store");
let request = objectStore.put(dataObject,key);
request.onsuccess = function () {
return data;
}
});
}
}
}
});
}