package mongodbcheckprofile;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.Mongo;
import java.io.IOException;
/**
* use ;
* db.setProfilingLevel(0);
* db.system.profile.drop()
* db.createCollection("system.profile", {capped:true, size:8000000})
* db.system.profile.stats()
* see http://www.mongodb.org/display/DOCS/Database+Profiler
* Query utils:
* db.system.profile.find( { millis : { $gt : 1 },ns : { $ne : 'nome_db.system.profile' } },{ millis:4, nscanned:1, nreturned:1, ns:1, op:1, query:1} )
* nel caso: che "nscanned" (molto maggiore) "nreturned" allora verificare gli indici sugli object elaborati.
*
* @author marco
*/
public class MongoDBCheckProfile {
static String host = "192.168.0.220";
static int port = 27017;
static String db_name = "nome_db";
static String op = "query";
static int maxtime = 4;
static int sleep = 5000;
public static void main(String[] args) throws IOException, InterruptedException {
while (true) {
System.out.println("start: " + System.currentTimeMillis());
Mongo m = new Mongo(host, port);
DB db = m.getDB(db_name);
BasicDBObject query = new BasicDBObject();
DBCollection coll = db.getCollection("system.profile");
query.put("millis", new BasicDBObject("$gt", 4));
query.put("ns", new BasicDBObject("$ne", db_name + ".system.profile"));
query.put("op", op);
DBCursor cur = coll.find(query);
while (cur.hasNext()) {
DBObject db_obj = cur.next();
System.out.println(db_obj.get("ts") + " - " + db_obj.get("ns") + " - " + db_obj.get("nscanned") + "/" + db_obj.get("nreturned") + " millis :" + db_obj.get("millis"));
}
Thread.sleep(sleep);
m.close();
}
}
}
package freedbtoMongoDb;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.Mongo;
public class Queries {
static String host = "mongodb1";
static int port = 27017;
static String db_name = "freedb";
public static void main(String[] args) {
try {
Mongo m = new Mongo(host, port);
DB db = m.getDB(db_name);
DBCollection coll = db.getCollection("disk");
BasicDBObject query = new BasicDBObject();
long t1 = System.currentTimeMillis();
query.put("artist", "Porcupine Tree");
DBCursor cur = coll.find(query).limit(0).skip(0);
long t2 = System.currentTimeMillis();
System.out.println("Tot time Find: " + (t2 - t1));
while (cur.hasNext()) {
System.out.println(cur.next());
}
long t3 = System.currentTimeMillis();
System.out.println("Tot time cursor: " + (t3 - t2));
m.close();
} catch (Exception e) {
System.out.println("Exception :" + e.getMessage());
}
}
}
Tot time Find: 2
{ "_id" : { "$oid" : "4e846f2330042d8d3130aad3"} , "id" : "blues_480f5d06" , "filename" : "blues/480f5d06" , "revision" : "7" , "title" : "The Sky Moves Sideways" , "genre" : "Rock" , "artist" : "Porcupine Tree" , "length" : "3935" , "year" : "1995" , "extd" : "YEAR: 1995 ID3G: 17" , "tracks" : { "3" : { "title" : "Prepare Yourself"} , "2" : { "title" : "The Moon Touches Your Shoulder"} , "1" : { "title" : "Dislocated Day"} , "0" : { "title" : "The Sky Moves Sideways (phase one)"} , "5" : { "title" : "The Sky Moves Sideways (phase two)"} , "4" : { "title" : "Moonloop"}}}}
etc..
etc..
Tot time cursor: 61270 (1 minuto abbondante)
BUILD SUCCESSFUL (total time: 1 minute 2 seconds)
package freedbtoMongoDb;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.Mongo;
public class Queries {
static String host = "mongodb1";
static int port = 27017;
static String db_name = "freedb";
public static void main(String[] args) {
try {
Mongo m = new Mongo(host, port);
DB db = m.getDB(db_name);
DBCollection coll = db.getCollection("disk");
BasicDBObject query = new BasicDBObject();
query.put("artist", "Porcupine Tree");
BasicDBObject field = new BasicDBObject();
field.put("title", "1");
long t1 = System.currentTimeMillis();
DBCursor cur = coll.find(query, field).limit(0).skip(0);
long t2 = System.currentTimeMillis();
System.out.println("Tot time Find: " + (t2 - t1));
while (cur.hasNext()) {
System.out.println(cur.next());
}
long t3 = System.currentTimeMillis();
System.out.println("Tot time cursor: " + (t3 - t2));
m.close();
} catch (Exception e) {
System.out.println("Exception :" + e.getMessage());
}
}
}
Tot time Find: 1
{ "_id" : { "$oid" : "4e846f2330042d8d3130aad3"} , "title" : "The Sky Moves Sideways"}
{ "_id" : { "$oid" : "4e846f2330042d8d3130add5"} , "title" : "The Sky Moves Sideways"}
{ "_id" : { "$oid" : "4e846f2330042d8d3130bb1a"} , "title" : "The Sky Moves Sideways"}
{ "_id" : { "$oid" : "4e846f5830042d8d3134caaa"} , "title" : "BBC Sessions 1993 & 1995"}
{ "_id" : { "$oid" : "4e846faa30042d8d31379b9d"} , "title" : "XM ReTracked"}
{ "_id" : { "$oid" : "4e846fba30042d8d3138cc14"} , "title" : "Lightbulb Sun"}
etc.
etc.
etc.
Tot time cursor: 73556
BUILD SUCCESSFUL (total time: 1 minute 14 seconds)
{
"id":"soundtrack_fe13d011",
"filename":"soundtrack/fe13d011",
"revision":"0",
"title":"100 Love Songs Stage 4 Cd4",
"genre":"Soundtrack",
"artist":"IndianDigitalAudio.com",
"length":"5074",
"year":"2011",
"extd":"",
"tracks":{
"15":{"title":"Mausam Achanak"},
"16":{"title":"I Dont Know What To Do"},
"13":{"title":"Aaja Maahi"},
"14":{"title":"Janeman"},
"11":{"title":"Roshan Dil Ka Jahan"},
"12":{"title":"Yaad Teri Aaye"},
"3":{"title":"Jo Ghumshuda"},
"2":{"title":"Dhadke Jiya"},
"1":{"title":"Nazrein Karam"},
"10":{"title":"Tum Bhi Ho Wahi"},
"0":{"title":"Aao Milo Chalo"},
"7":{"title":"Mausam"},
"6":{"title":"Mera Pehla Pehla Pyaar"},
"5":{"title":"Raat Ke Dhai Baje"},
"4":{"title":"Jaane Tu Meri Kya Hai"},
"9":{"title":"Pyaar Ki Yeh Kahani"},
"8":{"title":"Miss You Everyday"}
}
}
package freedbtoMongoDb;
import com.mongodb.DB;
import com.mongodb.DBObject;
import com.mongodb.Mongo;
import com.mongodb.util.JSON;
import java.io.IOException;
public class JsonToMongoDB {
static String host = "hostname";
static int port = 27017;
static String db_name = "freedb";
static String target_folder = "/marco/mongodb/file/";
public static void main(String[] args) throws IOException {
Mongo m = new Mongo(host, port);
DB db = m.getDB(db_name);
db.requestStart();
System.out.println("File: " + target_folder + "disk.json");
BigFile file = null;
try {
file = new BigFile(target_folder + "disk.json");
} catch (Exception ex) {
System.out.println("exception: " + ex.getMessage());
}
long ts = System.currentTimeMillis();
int i = 0;
for (String line : file) {
db.getCollection("disk").insert((DBObject) JSON.parse(line));
System.out.println("line: " + i);
i++;
}
System.out.println("tot time: " + (System.currentTimeMillis() - ts));
db.requestDone();
}
}
package orientdbtest;
import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.metadata.security.OUser;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
import com.orientechnologies.orient.server.OServer;
import com.orientechnologies.orient.server.OServerMain;
import java.io.File;
public class Main {
public static class GeoIp {
String from;
String to;
Long lat;
Long lng;
String country;
String state;
public void setFrom(String from) {
this.from = from;
}
public void setTo(String to) {
this.to = to;
}
public void setCountry(String country) {
this.country = country;
}
public void setState(String state) {
this.state = state;
}
public String getFrom() {
return from;
}
public String getTo() {
return to;
}
public String getCountry() {
return country;
}
public String getState() {
return state;
}
public Long getLng() {
return lng;
}
public Long getLat() {
return lat;
}
public void setLng(Long lng) {
this.lng = lng;
}
public void setLat(Long lat) {
this.lat = lat;
}
}
public static void main(String[] args) {
try {
String base = "/home/marco/Scrivania/orientdb/";
OServer server = OServerMain.create();
server.startup(new File(base + "/file/conf.xml"));
ODatabaseDocumentTx db = new ODatabaseDocumentTx("local:" + base + "/db").create();
try {
for (String s : db.getClusterNames()) {
System.out.println("name: " + s + " - " + db.countClusterElements(s));
}
db.begin();
BigFile geoip_list = new BigFile(base + "/file/data/GeoIPCountryWhois.csv");
long time = System.currentTimeMillis();
long c = 0;
for (String geoip_row : geoip_list) {
geoip_row = geoip_row.replaceAll("\"", "");
String[] s = geoip_row.split("\\,");
GeoIp geoIp = new GeoIp();
geoIp.setFrom(s[0]);
geoIp.setTo(s[1]);
geoIp.setLat(new Long(s[2]));
geoIp.setLng(new Long(s[3]));
geoIp.setCountry(s[4]);
geoIp.setState(s[5]);
ODocument doc = new ODocument(db);
doc.field("geoIp", geoIp);
doc.save();
if(c>10000 && c % 10000 == 0)
System.out.println(c);
c++;
}
db.commit();
System.out.println("tot time: " + (System.currentTimeMillis() - time));
System.out.println("commit:" + c);
} catch (Exception e) {
System.out.println("e:" + e.getMessage() + e.getStackTrace().toString());
db.rollback();
} finally {
for (String s : db.getClusterNames()) {
System.out.println("name: " + s + " - " + db.countClusterElements(s));
}
db.close();
}
server.shutdown();
} catch (Exception ex) {
System.out.println("ex:" + ex.getMessage()+ ex.getStackTrace().toString());
}
}
}
java -jar "/home/marco/netbeans-project/OrientDbTest/dist/OrientDbTest.jar"
2011-04-29 05:29:02:098 INFO [OServer] OrientDB Server v1.0rc1-SNAPSHOT is starting up...
2011-04-29 05:29:07:325 INFO [OServerNetworkListener] Listening binary connections on 0.0.0.0:2424
2011-04-29 05:29:12:331 INFO [OServerNetworkListener] Listening http connections on 0.0.0.0:2480
2011-04-29 05:29:12:331 INFO [OServer] OrientDB Server v1.0rc1-SNAPSHOT is active.name: internal - 4
name: index - 0
name: default - 0
name: orole - 3
name: ouser - 3
20000
30000
40000
...
...
...
690000
700000
710000
tot time: 39482
commit:719600
name: internal - 4
name: index - 0
name: default - 719600
name: orole - 3
name: ouser - 3
package orientdbtest;
import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.db.object.ODatabaseObjectTx;
import com.orientechnologies.orient.core.annotation.OVersion;
import com.orientechnologies.orient.core.metadata.security.OUser;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
import com.orientechnologies.orient.core.type.ODocumentWrapper;
import com.orientechnologies.orient.server.OServer;
import com.orientechnologies.orient.server.OServerMain;
import javax.persistence.Id;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
public class Main {
public static class GeoIp {
@Id
private Object id;
@OVersion
private Object version;
private String from;
private String to;
private Long lat;
private Long lng;
private String country;
private String state;
public void setFrom(String from) {
this.from = from;
}
public void setTo(String to) {
this.to = to;
}
public void setCountry(String country) {
this.country = country;
}
public void setState(String state) {
this.state = state;
}
public String getFrom() {
return from;
}
public String getTo() {
return to;
}
public String getCountry() {
return country;
}
public String getState() {
return state;
}
public Long getLng() {
return lng;
}
public Long getLat() {
return lat;
}
public void setLng(Long lng) {
this.lng = lng;
}
public void setLat(Long lat) {
this.lat = lat;
}
}
public static void main(String[] args) {
try {
String base = "/home/marco/Scrivania/orientdb/";
OServer server = OServerMain.create();
server.startup(new File(base + "/file/conf.xml"));
ODatabaseObjectTx db = new ODatabaseObjectTx("local:" + base + "/db").create();
db.getEntityManager().registerEntityClass(GeoIp.class);
try {
for (String s : db.getClusterNames()) {
System.out.println("name: " + s + " - " + db.countClusterElements(s));
}
db.begin();
BigFile geoip_list = new BigFile(base + "/file/data/GeoIPCountryWhois.csv");
long time = System.currentTimeMillis();
long c = 0;
for (String geoip_row : geoip_list) {
geoip_row = geoip_row.replaceAll("\"", "");
String[] s = geoip_row.split("\\,");
GeoIp geoIp = new GeoIp();
geoIp.setFrom(s[0]);
geoIp.setTo(s[1]);
geoIp.setLat(new Long(s[2]));
geoIp.setLng(new Long(s[3]));
geoIp.setCountry(s[4]);
geoIp.setState(s[5]);
db.save(geoIp);
if(c>10000 && c % 10000 == 0)
System.out.println(c);
c++;
}
db.commit();
System.out.println("tot time: " + (System.currentTimeMillis() - time));
System.out.println("commit:" + c);
} catch (Exception e) {
System.out.println("e:" + e.getMessage() + e.getStackTrace().toString());
db.rollback();
} finally {
for (String s : db.getClusterNames()) {
System.out.println("name: " + s + " - " + db.countClusterElements(s));
}
db.close();
}
server.shutdown();
} catch (Exception ex) {
System.out.println("ex:" + ex.getMessage()+ ex.getStackTrace().toString());
}
}
}