Real time stock market prediction
This article contains the source code for my video, and also the link to the dataset
I have developed the entire server-side principal architecture for real-time stock market prediction with Machine Learning. I have used Tensorflow.js for constructing ml model architecture, and Kafka for real-time data streaming and pipelining.
Let me show you the folder structure.
You should write the code yourself, to understand what it does, so now i am going to show you the code:
config.js
const config = {
"kafkaConfig":{
KafkaHost:'localhost:9092',
KafkaTopic1: 'StockMarketAnalysis',
KafkaTopic2: 'StockMarketPredictions',
},
"dbConfig":{
user: 'VictorBasu',
password: 'V1cTOR',
server: 'clustera1.cvglu.mongodb.net/',
database: 'StockLogsDB',
collectionStream: 'StockData',
collectionML: 'mlPrediction',
},
"trainConfig":{
trainSize: 85,
epoch: 50,
batchSize: 32,
modelDir: 'models',
},
"type": "Open"
};
module.exports = config;
consumer.js
const kafka = require('kafka-node');
const config = require('./config');
const db = require('./InstantiateDB');
const configKafka = config.kafkaConfig;
const Consumer = kafka.Consumer;
const client = new kafka.KafkaClient({idleConnection: 24 * 60 * 60 * 1000, kafkaHost: configKafka.KafkaHost});
let consumer = new Consumer(
client,
[{topic: configKafka.KafkaTopic1, partition: 0 }],
{
autoCommit: true,
fetchMaxWaitMs: 1000,
fetchMaxBytes: 1024 * 1024,
encoding: 'utf8',
// fromOffset: false
}
);
consumer.on('message', async function(message){
// Storing or updating consumed stream messages to MongoDB.
db.updateMongoDB(message)
});
consumer.on('error', function(error) {
// handle error
console.log('error', error);
});
createTopics.js
const kafka = require('kafka-node');
const config = require('./config');
const client = new kafka.KafkaClient({kafkaHost: config.KafkaHost});
const topicToCreate = [{
topic: config.KafkaTopic1,
partitions: 1,
replicationFactor: 3,
},
{
topic: config.KafkaTopic2,
partitions: 1,
replicationFactor: 3,
}
]
client.createTopics(topicToCreate, (error, result) => {
// result is an array of any errors if a given topic could not be created
console.log(result, 'topic created successfully');
})
The rest of the code is under a paid wall. Please consider subscribing, you get access to all of the source code and articles and many other things. It also helps me pay my tuition fees. So please if you can then please do subscribe, or visit Medium, By Clicking here for free articles