MBFastUrl - nuova beta - vari fix - nuove stats

- cron con statistiche per ip
- fix sulla grafica
- varie ottimizzazione sulle conf
- stats log sui qrcode
- cron e stats con sommatoria degli ip nelle varie fasi view,save,qrcode

il server di test attualmente è arrivato ad avere 460 Link salvati e più di 12500 click sui link, non male visto che lo utilizzo solo io per propagare i link sui vari social.

per ora il db dei link in SqLite è 26 MB invece quello di elaborazione delle statistiche 19 MB.

Per ora SqLite tiene, magari prossimamente ci sarà un porting a OrientDB.

From Freedb.org to OrientDB - #4

#1 - #2 - #3

Download a big file about 170 Mb.
Uncompress  ... about 614 Mb and 2630481 file...

to speed up the importation are processed only 1000 files per folder.

sample porting schema from FreeDB to OrientDB:

import code

import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.metadata.schema.OProperty.INDEX_TYPE;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
import com.orientechnologies.orient.server.OServer;
import com.orientechnologies.orient.server.OServerMain;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import org.apache.commons.io.FileUtils;

private String base = "/Users/marco/orientdb/";
private String root_dir = base + "/file_freedb/freedb-complete-20090101/";
private HashMap cache_year = new HashMap();

private HashMap cache_genre = new HashMap();

  private void import_data() {

        try {

            OServer server = OServerMain.create();
            server.startup(new File(base + "/file/conf.xml"));

            ODatabaseDocumentTx db = new ODatabaseDocumentTx("local:" + base + "/freedb");
            if (!db.exists()) {
                db.create();
                System.out.println("create new DB");
            } else {
                db.delete();
                db.create();
                System.out.println("delete and create new DB");
            }


            FileFilter directoryFilter = new FileFilter() {

                public boolean accept(File file) {
                    return file.isDirectory();
                }
            };

            //default index on odocument
            db.begin();


            ODocument oArtist = new ODocument(db, "artist");
            oArtist.field("name", "Various", OType.STRING);
            oArtist.save();

            db.getMetadata().getSchema().getClass("artist").createProperty("name", OType.STRING).createIndex(INDEX_TYPE.FULLTEXT);
            db.getMetadata().getSchema().save();


            ODocument oTrack = new ODocument(db, "track");
            oTrack.field("title", "Various", OType.STRING);
            oTrack.save();

            db.getMetadata().getSchema().getClass("track").createProperty("title", OType.STRING).createIndex(INDEX_TYPE.FULLTEXT);
            db.getMetadata().getSchema().save();


            ODocument oGendr = new ODocument(db, "genre");
            oGendr.field("name", "Various", OType.STRING);
            oGendr.save();

            db.getMetadata().getSchema().getClass("genre").createProperty("name", OType.STRING).createIndex(INDEX_TYPE.UNIQUE);
            db.getMetadata().getSchema().save();


            ODocument oYear = new ODocument(db, "year");
            oYear.field("data", "19000101", OType.DATE);
            oYear.save();

            db.getMetadata().getSchema().getClass("year").createProperty("data", OType.DATE).createIndex(INDEX_TYPE.UNIQUE);
            db.getMetadata().getSchema().save();


            db.commit();


            File[] dirs = new File(root_dir).listFiles(directoryFilter);
            int i = 0;
            for (File dir : dirs) {

                if (!dir.isDirectory()) {
                    continue;
                }

                System.out.println("" + dir);

                int max_file_for_debug = 0;
                Collection files = FileUtils.listFiles(dir, null, true);
                for (File file : files) {
                    if (file.getName().startsWith(".")) {
                        continue;
                    }


                    try {
                        List lines = FileUtils.readLines(file);

                        db.begin();
                        ODocument oDisk = new ODocument(db, "disk");

                        ArrayList tracks = new ArrayList();

                        String titles = "";
                        String extd = "";
                        for (String line : lines) {

                            if (line.startsWith("# Disc length:")) {
                                String length = line.replaceAll("# Disc length:", "").replaceAll("seconds", "").replaceAll("secs", "").trim();

                                oDisk.field("Disc Length", length, OType.INTEGER);
                            }

                            if (line.startsWith("# Revision:")) {
                                String revision = line.replaceAll("# Revision:", "").trim();


                                oDisk.field("revision", revision, OType.INTEGER);
                            }


                            if (line.startsWith("#")) {
                                continue;
                            }

                            String ele[] = line.split("=");

                            if (ele == null || ele.length == 1) {
                                continue;
                            }

                            String key = ele[0];
                            String value = ele[1];


                            if (key.equals("DISKID")) {
                                oDisk.field(key.toLowerCase(), value, OType.STRING);
                            }

                            if (key.equals("DYEAR")) {
                                oDisk.field("year", check_and_create_year(value + "0101", db), OType.LINK);
                            }

                            if (key.equals("DGENRE")) {

                                oDisk.field("genre", check_and_create_genre(value, db), OType.LINK);
                            }

                            //concatenate multiple title lines
                            if (key.equals("DTITLE")) {
                                titles += value;
                            }

                            if (key.equals("EXTD")) {
                                extd += value;
                            }



                            //tracks list
                            if (key.startsWith("TTITLE")) {
                                oTrack = new ODocument(db, "track");
                                oTrack.field("n", key.replaceAll("TTITLE", ""), OType.INTEGER);

                                String tartist = "";


                                oTrack.field("title", getTitle(value));

                                tartist = getAuthor(value);

                                if (!tartist.equals("")) {

                                    oTrack.field("artist", check_and_create_artist(tartist, db), OType.LINK);
                                } else {
                                    oTrack.field("artist");
                                }


                                oTrack.save();
                                tracks.add(oTrack);

                            }

                        }

                        //add track_list
                        if (!tracks.isEmpty()) {
                            oDisk.field("tracks", tracks, OType.EMBEDDEDLIST);
                        }

                        //title and artist disk
                        if (!titles.equals("")) {

                            oDisk.field("title", getTitle(titles));
                            oDisk.field("artist", check_and_create_artist(getAuthor(titles), db), OType.LINK);

                        }

                        //title and artist disk
                        if (!extd.equals("")) {
                            oDisk.field("extd", extd);
                        }

                        oDisk.save();
                        db.commit();

                    } catch (IOException ex) {
                        System.out.println("ex (1):" + ex.getMessage() + ex.getStackTrace().toString());

                        for (StackTraceElement s : ex.getStackTrace()) {
                            System.out.println("" + s);
                        }

                        db.rollback();
                        continue;

                    }


                    i++;
                    if ((i >= 1000) && (i % 1000) == 1) {
                        System.out.println("\t" + file);
                        for (String s : db.getClusterNames()) {
                            System.out.println("cluster: " + s + " - " + db.countClusterElements(s));

                        }

                        //for debug max 1000 file for folder
                        break;
                    }

                }

            }


            db.close();

            //server
            server.shutdown();
        } catch (Exception ex) {
            System.out.println("ex (2):" + ex.getMessage() + ex.getStackTrace().toString());


            for (StackTraceElement s : ex.getStackTrace()) {
                System.out.println("" + s);
            }

        }

    }

    private ODocument check_and_create_artist(String name, ODatabaseDocumentTx db) {


        if (name.equals("")) {
            return null;
        }


        OSQLSynchQuery query = new OSQLSynchQuery("select from artist where name = ?");
        List result = db.command(query).execute(name);


        if (!result.isEmpty()) {
            return (ODocument) result.get(0);

        } else {

            ODocument oArtist = new ODocument(db, "artist");
            oArtist.field("name", /*a*/ name, OType.STRING);
            oArtist.save();

            return oArtist;
        }


    }

    private ODocument check_and_create_year(String year, ODatabaseDocumentTx db) {

        if (cache_year.containsKey(year)) {
            return cache_year.get(year);
        }


        OSQLSynchQuery query = new OSQLSynchQuery("select from year where data = ?");
        List result = db.command(query).execute(year);


        if (!result.isEmpty()) {
            cache_year.put(year, (ODocument) result.get(0));
            return (ODocument) result.get(0);

        } else {

            ODocument oYear = new ODocument(db, "year");
            oYear.field("data", year, OType.DATE);
            oYear.save();
            cache_year.put(year, oYear);

            return oYear;
        }


    }

    private ODocument check_and_create_genre(String genre, ODatabaseDocumentTx db) {

        if (cache_genre.containsKey(genre)) {
            return cache_genre.get(genre);
        }

        OSQLSynchQuery query = new OSQLSynchQuery("select from genre where name = ?");
        List result = db.command(query).execute(genre);

        if (!result.isEmpty()) {
            cache_genre.put(genre, (ODocument) result.get(0));
            return (ODocument) result.get(0);

        } else {

            ODocument oGenre = new ODocument(db, "genre");
            oGenre.field("name", genre, OType.STRING);
            oGenre.save();
            cache_genre.put(genre, oGenre);
            return oGenre;
        }


    }

    private String getTitle(String value) {


        if (value.indexOf("/") == -1) {
            return escape(value);
        }

        try {
            return escape(value.split("/")[0]);
        } catch (Exception e) {
            return "";
        }

    }

    private String getAuthor(String value) {

        if (value.indexOf("/") == -1) {
            return "";
        }

        if (value.split("/").length == 0) {
            return "";
        }

        try {
            return escape(value.split("/")[1]);
        } catch (Exception e) {
            return "";
        }


    }

    private String escape(String s) {
        if (s == null) {
            return s;
        }
        return s.trim().replaceAll("\\[", "").replaceAll("\\]", "").replaceAll("'", "\\\\'");
    }

Lib:

• commons-io-2.0.1.jar
• orient-commons-1.0rc2-SNAPSHOT.jar
• orientdb-client-1.0rc2-SNAPSHOT.jar
• orientdb-core-1.0rc2-SNAPSHOT.jar
• orientdb-enterprise-1.0rc2-SNAPSHOT.jar
• orientdb-server-1.0rc2-SNAPSHOT.jar
• orientdb-tools-1.0rc2-SNAPSHOT.jar
• persistence-api-1.0.jar

after several hours...

cluster: internal - 3
cluster: index - 882
cluster: default - 0
cluster: orole - 3
cluster: ouser - 3
cluster: artist - 43767
cluster: track - 155602
cluster: genre - 995
cluster: year - 114
cluster: disk - 11001

about 4GB of db...

Test Query:


first time:
query:select from artist name like 'Pink%' tot time:25692 ms
next:
query:select from artist name like 'Pink%' tot time:1646 ms

first time:
query:select from disk where artist.name like 'Pink%' tot time: 13388 ms
next:
query:select from disk where artist.name like 'Pink%' tot time: 4714 ms

first time:
query:select from disk where tracks contains ( artist.name like 'Pink%' ) tot time: 1628 ms
next:
query:select from disk where tracks contains ( artist.name like 'Pink%' ) tot time: 1481 ms

first/next time:
query:select from disk where year.data = '19780101' tot time: 906

download code

#1 - #2 - #3

linux - conteggio file via bash

conteggio dei file con la bash di linux in modo ricorsivo.

find . | wc -l

[via]

java - Directory and File listing

Java - Generare e scaricare il Qrcode di Google

comoda funzionalità per scaricare da url un file generato da google per di Qrcode utilizzando le commons.io:

package com.utils;

import java.io.File;
import java.io.IOException;
import java.net.URL;

public class HttpUtil {

    public static synchronized void downloadData(String url, String target) {

        try {
            org.apache.commons.io.FileUtils.copyURLToFile(new URL(url), new File(target));
        } catch (IOException ex) {
            System.err.println("errore " + ex.getMessage());
        }


    }
}

poi semplicemente:

HttpUtil.downloadData("https://chart.googleapis.com/chart?cht=qr&chs=100x100&chl=http://www.google.it", "/home/marco/qrcode.png");

la documentazione di Google

OrientDB - Import da csv relazionali, relazioni, archiviare file e query #3

Proseguo la panoramica sulle funzionalità di OrientDb. Il miglior metodo per capire come funziona è quello di sperimentare. Genero dei dati test per simulare il funzionamento di un db relazionale social su file csv. Così costituito:

Utenti: elenco di utenti  id;nome;cognome;mail;password;  con relativo avatar collegato all'id.

lnk_utenti: collegamento tramite id degli utenti sugli utenti ( tipo amici).

post: id;utente_id;file;text;data;like[utenti];commenti[commento];tag[utenti]; dove i campi compresi tra [] identifica una lista di collegamenti.

commenti: id;utente_id;text;data;like[utenti];

Questa procedura mi permette di provare:

• salvataggio in blocco dell'object
• salvataggio nel db del file (avatar) vedi doc                         
• collegamenti N:M su liste

package orientdbtest;

import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.record.impl.ORecordBytes;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
import com.orientechnologies.orient.server.OServer;
import com.orientechnologies.orient.server.OServerMain;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.io.FileUtils;

public class Social {

    public static void main(String[] args) {
        try {

            String base = "/Users/marco/orientdb/";

            OServer server = OServerMain.create();
            server.startup(new File(base + "/file/conf.xml"));

            ODatabaseDocumentTx db = new ODatabaseDocumentTx("local:" + base + "/social").create();


            //elenco dei cluster
            for (String s : db.getClusterNames()) {
                System.out.println(s + " - " + db.countClusterElements(s));

            }



            //import UTENTI
            try {
                System.out.println("**** Start Import Utenti");
                long time = System.currentTimeMillis();

                BigFile big_file = new BigFile(base + "/file_social/utente.csv");

                boolean first_line = true;

                db.begin();

                for (String utente : big_file) {
                    if (first_line) {
                        first_line = false;
                        continue;
                    }

                    String[] utente_split = utente.split("\\;");

                    ODocument Outente = new ODocument(db, "Utenti");
                    Outente.field("id", utente_split[0], OType.SHORT);
                    Outente.field("nome", utente_split[1], OType.STRING);
                    Outente.field("cognome", utente_split[2], OType.STRING);
                    Outente.field("email", utente_split[3], OType.STRING);
                    Outente.field("password", utente_split[4], OType.STRING);

                    try {
                        File avatar = new File(base + "/file_social/avatar/" + utente_split[0] + ".jpg");

                        if (avatar.exists()) {
  
                            Outente.field("avatar", new ORecordBytes(db, FileUtils.readFileToByteArray(new File(base + "/file_social/avatar/" + utente_split[0] + ".jpg"))));
                            System.out.println("immagine trovata: " + base + "/file_social/avatar/" + utente_split[0] + ".jpg");
                        } else {
                            System.out.println("immagine " + base + "/file_social/avatar/" + utente_split[0] + ".jpg" + " non trovata");
                        }
                    } catch (IOException eFile) {
                        System.out.println("immagine " + base + "/file_social/avatar/" + utente_split[0] + ".jpg" + " non trovata");
                    }
                    Outente.save();

                }

                db.commit();

                System.out.println("tot time: " + (System.currentTimeMillis() - time));

            } catch (Exception e) {
                System.out.println("e1:" + e.getMessage() + e.getStackTrace().toString());
                db.rollback();
            }



            //import AMICI UTENTI

            try {
                System.out.println("**** Start Import Amici");


                long time = System.currentTimeMillis();


                BigFile big_file = new BigFile(base + "/file_social/lnk_utenti.csv");

                boolean first_line = true;



                for (String lnk_utenti : big_file) {
                    if (first_line) {
                        first_line = false;
                        continue;
                    }

                    String[] lnk_utenti_split = lnk_utenti.split("\\;");

                    db.begin();
                    //ricerco gli utente correlati e li aggancio uno agli altri
                    ODocument utente1 = (ODocument) db.query(new OSQLSynchQuery("select from utenti where id = '" + lnk_utenti_split[0] + "'")).get(0);
                    ODocument utente2 = (ODocument) db.query(new OSQLSynchQuery("select from utenti where id = '" + lnk_utenti_split[1] + "'")).get(0);

                    ArrayList lnk = new ArrayList();

                    if (utente1.field("lnk") != null) {
                        lnk = (ArrayList) utente1.field("lnk");
                    } else {
                        lnk = new ArrayList();
                    }

                    //verifico se nei lnk esiste già l'utente
                    boolean trovato = false;
                    if (!lnk.isEmpty()) {
                        for (ODocument check : lnk) {
                            System.out.println("check utente-1 id:" + check.field("id").toString() + " to: " + utente2.field("id").toString());
                            if (check.field("id").toString().equals(utente2.field("id").toString())) {
                                trovato = true;
                                continue;
                            }

                        }
                    }

                    if (!trovato) {

                        lnk.add(utente2);
                        utente1.field("lnk", lnk, OType.EMBEDDEDLIST);
                        System.out.println("utente-1 add: " + utente2.field("id"));
                        utente1.save();
                    }



                    if (utente2.field("lnk") != null) {
                        lnk = (ArrayList) utente2.field("lnk");
                    } else {
                        lnk = new ArrayList();
                    }


                    //verifico se nei lnk esiste già l'utente
                    trovato = false;
                    if (!lnk.isEmpty()) {
                        for (ODocument check : lnk) {
                            System.out.println("check utente-2 id:" + check.field("id").toString() + " to: " + utente1.field("id").toString());
                            if (check.field("id").toString().equals(utente1.field("id").toString())) {
                                trovato = true;
                                continue;
                            }

                        }
                    }
                    if (!trovato) {
                        lnk.add(utente1);
                        System.out.println("utente-2 add: " + utente1.field("id"));
                        utente2.field("lnk", lnk, OType.EMBEDDEDLIST);
                        utente2.save();
                    }


                    db.commit();

                }




                System.out.println("tot time: " + (System.currentTimeMillis() - time));


            } catch (Exception e) {
                System.out.println("e2:" + e.getMessage() + e.getStackTrace().toString());
                db.rollback();
            }


            for (String s : db.getClusterNames()) {
                System.out.println(s + " - " + db.countClusterElements(s));

            }




            try {
                System.out.println("**** Start Import Post");

                long time = System.currentTimeMillis();


                BigFile big_file = new BigFile(base + "/file_social/post.csv");

                boolean first_line = true;

                db.begin();

                for (String post : big_file) {
                    if (first_line) {
                        first_line = false;
                        continue;
                    }

                    String[] post_split = post.split("\\;");


                    //ricerco gli utente correlati e li aggancio uno agli altri
                    ODocument utente = (ODocument) db.query(new OSQLSynchQuery("select from utenti where id = '" + post_split[1] + "'")).get(0);

                    ODocument OPost = new ODocument(db, "Post");
                    OPost.field("id", post_split[0], OType.SHORT);
                    OPost.field("utente", utente, OType.EMBEDDED);
                    OPost.field("file", post_split[2], OType.STRING);
                    OPost.field("text", post_split[3], OType.STRING);
                    OPost.field("data", post_split[4], OType.STRING);

                    if (!post_split[5].equals("-")) {
                        String[] split_utenti = post_split[5].split("\\,");
                        ArrayList lnk = new ArrayList();
                        for (String ut : split_utenti) {
                            lnk.add((ODocument) db.query(new OSQLSynchQuery("select from utenti where id = '" + ut + "'")).get(0));
                        }
                        OPost.field("lnk", lnk, OType.EMBEDDEDLIST);
                    }


                    if (!post_split[7].equals("-")) {
                        String[] split_utenti = post_split[7].split("\\,");

                        ArrayList tag = new ArrayList();
                        for (String ut : split_utenti) {
                            tag.add((ODocument) db.query(new OSQLSynchQuery("select from utenti where id = '" + ut + "'")).get(0));
                        }
                        OPost.field("tag", tag, OType.EMBEDDEDLIST);
                    }


                    OPost.save();

                }

                db.commit();

                System.out.println("tot time: " + (System.currentTimeMillis() - time));


            } catch (Exception e) {
                System.out.println("e3:" + e.getMessage() + e.getStackTrace());
                db.rollback();
            }



            try {
                System.out.println("**** Start Import commenti");

                long time = System.currentTimeMillis();


                BigFile big_file = new BigFile(base + "/file_social/commenti.csv");

                boolean first_line = true;

                db.begin();

                for (String commenti : big_file) {
                    if (first_line) {
                        first_line = false;
                        continue;
                    }

                    String[] commenti_split = commenti.split("\\;");


                    //ricerco gli utente correlati e li aggancio uno agli altri
                    ODocument utente = (ODocument) db.query(new OSQLSynchQuery("select from utenti where id = '" + commenti_split[1] + "'")).get(0);

                    ODocument OCommenti = new ODocument(db, "commenti");
                    OCommenti.field("id", commenti_split[0], OType.SHORT);
                    OCommenti.field("utente", utente, OType.EMBEDDED);
                    OCommenti.field("text", commenti_split[2], OType.STRING);
                    OCommenti.field("data", commenti_split[3], OType.STRING);


                    if (!commenti_split[4].equals("-")) {

                        String[] split_utenti = commenti_split[4].split("\\,");

                        ArrayList lnk = new ArrayList();
                        for (String ut : split_utenti) {
                            lnk.add((ODocument) db.query(new OSQLSynchQuery("select from utenti where id = '" + ut + "'")).get(0));
                        }
                        OCommenti.field("lnk", lnk, OType.EMBEDDEDLIST);
                    }

                    OCommenti.save();

                }

                db.commit();

                System.out.println("tot time: " + (System.currentTimeMillis() - time));


            } catch (Exception e) {
                System.out.println("e5:" + e.getMessage() + e.getStackTrace());
                db.rollback();
            }



            try {
                System.out.println("**** Start Add Commenti/Utenti to Post");

                long time = System.currentTimeMillis();


                BigFile big_file = new BigFile(base + "/file_social/post.csv");

                boolean first_line = true;

                db.begin();

                for (String post : big_file) {
                    if (first_line) {
                        first_line = false;
                        continue;
                    }

                    String[] post_split = post.split("\\;");


                    //ricerco gli utente correlati e li aggancio uno agli altri
                    ODocument oPost = (ODocument) db.query(new OSQLSynchQuery("select from post where id = '" + post_split[0] + "'")).get(0);



                    if (!post_split[6].equals("-")) {

                        String[] split_commenti = post_split[6].split("\\,");
                        ArrayList lnk = new ArrayList();

                        for (String ut : split_commenti) {
                            lnk.add((ODocument) db.query(new OSQLSynchQuery("select from commenti where id = '" + ut + "'")).get(0));
                        }

                        oPost.field("commenti", lnk, OType.EMBEDDEDLIST);
                        oPost.save();
                    }



                }

                db.commit();

                System.out.println("tot time: " + (System.currentTimeMillis() - time));


            } catch (Exception e) {
                System.out.println("e3:" + e.getMessage() + e.getStackTrace());
                db.rollback();
            }



            try {
                System.out.println("**** Rimuovo tutti gli id");

                long time = System.currentTimeMillis();


                db.begin();

                String obj[] = {"utenti", "post", "commenti"};
                for (String o : obj) {
                    List docs = db.query(new OSQLSynchQuery("select from " + o));

                    for (ODocument doc : docs) {
                        doc.removeField("id");
                        doc.save();
                    }
                }

                db.commit();

                System.out.println("tot time: " + (System.currentTimeMillis() - time));


            } catch (Exception e) {
                System.out.println("e4:" + e.getMessage() + e.getStackTrace());
                db.rollback();
            }




            for (String s : db.getClusterNames()) {
                System.out.println(s + " - " + db.countClusterElements(s));

            }


            db.close();
            server.shutdown();
            
        } catch (Exception ex) {
            System.out.println("ex:" + ex.getMessage() + ex.getStackTrace().toString());
        }

        
        
    }
}

conclusa questa operazione di import delle info procedo ad eseguire qualche query per sperimentare il tutto.

con una query sola molto veloce riesco a:

• estrarre carlo dall'elenco in base alla email
• estrarre gli amici di carlo (prelevare l'avatar e salvarlo su disco)
• estrarre tutti i post di un fede e ricavare tutti i commenti e like

package orientdbtest;

import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.record.impl.ORecordBytes;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
import com.orientechnologies.orient.server.OServer;
import com.orientechnologies.orient.server.OServerMain;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.io.FileUtils;

public class Main {

 
    public static void main(String[] args) {
        try {

            String base = "/Users/marco/orientdb/";

            OServer server = OServerMain.create();
            ODatabaseDocumentTx db = new ODatabaseDocumentTx("local:" + base + "/social").open("admin", "admin");

            try {

                System.out.println("****** Start Query");

                long time = System.currentTimeMillis();

                ODocument carlo = (ODocument) db.query(new OSQLSynchQuery("select from utenti where email = 'carlo@email.it'")).get(0);
                System.out.println("Query dati di carlo - tot time: " + (System.currentTimeMillis() - time));
                System.out.println("nome:" + carlo.field("nome"));
                System.out.println("cognome:" + carlo.field("cognome"));

                //scrivo su file l'avatar presente, qui si può decidere cosa fare mandarlo sulla response o altro...
                File avatar = new File(base + "/test_avatar_di_carlo.jpg");
                FileUtils.writeByteArrayToFile(avatar, ((ORecordBytes) carlo.field("avatar")).toStream());
                System.out.println("Avatar estratto in:" + avatar);


                time = System.currentTimeMillis();
                List amici = db.query(new OSQLSynchQuery("select flatten(lnk) from utenti where email = 'carlo@email.it'"));
                System.out.println("Query elenco amici di carlo tot time: " + (System.currentTimeMillis() - time));
                for (ODocument amico : amici) {
                    System.out.println(amico.field("email"));
                }

                System.out.println();
                System.out.println();

                time = System.currentTimeMillis();
                amici = db.query(new OSQLSynchQuery("select flatten(lnk) from utenti where email = 'roberta@email.it'"));
                System.out.println("Query amici di amici di roberta - tot time: " + (System.currentTimeMillis() - time));


                for (ODocument amico : amici) {
                    System.out.print(amico.field("email") + " : ");

                    for (ODocument amico_di_amico : ((ArrayList) amico.field("lnk", OType.EMBEDDEDLIST))) {
                        System.out.print(amico_di_amico.field("email") + " ");
                    }

                    System.out.println();

                }



                System.out.println();
                System.out.println();





                time = System.currentTimeMillis();
                List mia_bacheca = db.query(new OSQLSynchQuery("select  * from post where utente.email = 'fede@email.it'"));
                System.out.println("Query la bacheca di fede tot time: " + (System.currentTimeMillis() - time));


                for (ODocument post : mia_bacheca) {
                    System.out.println("post: '" + post.field("text") + "' da " + ((ODocument) post.field("utente")).field("email"));
                    if (post.field("lnk") != null) {
                        ArrayList lnks = (ArrayList) post.field("lnk");
                        for (ODocument lnk : lnks) {
                            System.out.println("\t\tquesto post piace a: " + lnk.field("email"));
                        }
                    }


                    if (post.field("commenti") != null) {

                        ArrayList commenti = (ArrayList) post.field("commenti");
                        for (ODocument commento : commenti) {
                            System.out.println("\tcommento: '" + commento.field("text") + "' da " + ((ODocument) commento.field("utente")).field("email"));
                            if (commento.field("lnk") != null) {
                                ArrayList lnks = (ArrayList) commento.field("lnk");
                                for (ODocument lnk : lnks) {
                                    System.out.println("\t\tquesto commento piace a: " + lnk.field("email"));
                                }
                            }


                        }
                    }

                    System.out.println();

                }


                System.out.println("******* Stop Queries");


            } catch (Exception e) {
                System.out.println("e:" + e.getMessage() + e.getStackTrace().toString());
                db.rollback();
            } finally {


                db.close();
            }


            //server
            server.shutdown();
        } catch (Exception ex) {
            System.out.println("ex:" + ex.getMessage() + ex.getStackTrace().toString());
        }


    }
}

risultato:

****** Start Query
Query dati di carlo - tot time: 56
nome:carlo
cognome:rossi
Avatar estratto in:/Users/marco/orientdb/test_avatar_di_carlo.jpg
Query elenco amici di carlo tot time: 38
mauro@email.it
matteo@email.it
roberta@email.it
marco@email.it


Query amici di amici di roberta - tot time: 27
carlo@email.it : mauro@email.it matteo@email.it roberta@email.it 
mauro@email.it : carlo@email.it fede@email.it roberta@email.it 
fede@email.it : mauro@email.it roberta@email.it 


Query la bacheca di fede tot time: 56
post: 'una buona giornata a carlo e roberta' da fede@email.it
		questo post piace a: mauro@email.it
		questo post piace a: matteo@email.it
	commento: 'buona giornata a te' da mauro@email.it
		questo commento piace a: fede@email.it
		questo commento piace a: carlo@email.it
	commento: 'buona giornata a voi' da roberta@email.it
		questo commento piace a: carlo@email.it
		questo commento piace a: roberta@email.it

******* Stop Queries

download esempio completo

note:
le query non funzionano così : email='carlo@email.it'. Ma così  email = 'carlo@email.it' (attenzione agli spazi)

altre puntate - #1 #2

MBFastUrl - nuova beta - fix e prima parte di admin

Aggiornamento del progetto MBFastUrl:

- Home page made in aurora.
- Pagina di login con relativo login (demo/demo)
- Prima parte di admin
- vari fix

OrientDB - Metodi di scrittura: ODocument e Pojo (Embedding in java) - #2

altre puntate: #1

prelevo un file di 10 mb da questo sito per avere dei dati di test per provare le performace di scrittura di questo db, ovvero prendo una classe la aggiungo ad un document e salvo il tutto.

Il file era troppo piccolo quindi lo copia e incollato su se stesso diverse volte ottenendo circa 720000  righe (adesso il mio file pesa 52 MB circa).

metoto ODocument con la classe ODatabaseDocumentTx

ecco il codice:

package orientdbtest;

import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.metadata.security.OUser;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
import com.orientechnologies.orient.server.OServer;
import com.orientechnologies.orient.server.OServerMain;
import java.io.File;

public class Main {

    public static class GeoIp {

        String from;
        String to;
        Long lat;
        Long lng;
        String country;
        String state;

        public void setFrom(String from) {
            this.from = from;
        }

        public void setTo(String to) {
            this.to = to;
        }

        public void setCountry(String country) {
            this.country = country;
        }

        public void setState(String state) {
            this.state = state;
        }

        public String getFrom() {
            return from;
        }

        public String getTo() {
            return to;
        }

        public String getCountry() {
            return country;
        }

        public String getState() {
            return state;
        }

        public Long getLng() {
            return lng;
        }

        public Long getLat() {
            return lat;
        }

        public void setLng(Long lng) {
            this.lng = lng;
        }

        public void setLat(Long lat) {
            this.lat = lat;
        }
    }

    public static void main(String[] args) {
        try {

            String base = "/home/marco/Scrivania/orientdb/";

            OServer server = OServerMain.create();
            server.startup(new File(base + "/file/conf.xml"));

            ODatabaseDocumentTx db = new ODatabaseDocumentTx("local:" + base + "/db").create();


            try {


                for (String s : db.getClusterNames()) {
                    System.out.println("name: " + s + " - " + db.countClusterElements(s));

                }

                db.begin();


                BigFile geoip_list = new BigFile(base + "/file/data/GeoIPCountryWhois.csv");


                long time = System.currentTimeMillis();


                long c = 0;
                for (String geoip_row : geoip_list) {
                    geoip_row = geoip_row.replaceAll("\"", "");
                    String[] s = geoip_row.split("\\,");
                    GeoIp geoIp = new GeoIp();
                    geoIp.setFrom(s[0]);
                    geoIp.setTo(s[1]);
                    geoIp.setLat(new Long(s[2]));
                    geoIp.setLng(new Long(s[3]));
                    geoIp.setCountry(s[4]);
                    geoIp.setState(s[5]);

                    ODocument doc = new ODocument(db);
                    doc.field("geoIp", geoIp);
                    doc.save();
                    if(c>10000 && c % 10000 == 0)
                        System.out.println(c);
                    c++;
                }



                db.commit();
                System.out.println("tot time: " + (System.currentTimeMillis() - time));
                System.out.println("commit:" + c);


            } catch (Exception e) {
                System.out.println("e:" + e.getMessage() + e.getStackTrace().toString());
                db.rollback();
            } finally {


                for (String s : db.getClusterNames()) {
                    System.out.println("name: " + s + " - " + db.countClusterElements(s));

                }

                db.close();
            }

            server.shutdown();
        } catch (Exception ex) {
            System.out.println("ex:" + ex.getMessage()+ ex.getStackTrace().toString());
        }


    }
}

qui la classe BigFile

lancio il tutto da console per evitare di usare troppa ram dalla ide visto che leggo il file con un iterator.

java -jar "/home/marco/netbeans-project/OrientDbTest/dist/OrientDbTest.jar" 

2011-04-29 05:29:02:098 INFO [OServer] OrientDB Server v1.0rc1-SNAPSHOT is starting up...
2011-04-29 05:29:07:325 INFO [OServerNetworkListener] Listening binary connections on 0.0.0.0:2424
2011-04-29 05:29:12:331 INFO [OServerNetworkListener] Listening http connections on 0.0.0.0:2480
2011-04-29 05:29:12:331 INFO [OServer] OrientDB Server v1.0rc1-SNAPSHOT is active.name: internal - 4
name: index - 0
name: default - 0
name: orole - 3
name: ouser - 3
20000
30000
40000
...
...
...
690000
700000
710000
tot time: 39482
commit:719600
name: internal - 4
name: index - 0
name: default - 719600
name: orole - 3
name: ouser - 3

...non male!!! inserite 719600 righe in 39 secondi tutto embeddato in un jar sul pc client....

se utilizzo il metodo POJO, ovvero OrientDB mappa la classe e salvo direttamente la classe mappata il tutto diventa molto più lento (forse perchè deve ricostruire tutto l'object in fase di salvataggio).

metodo POJO con la class ODatabaseObjectTx

package orientdbtest;

import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.db.object.ODatabaseObjectTx;
import com.orientechnologies.orient.core.annotation.OVersion;
import com.orientechnologies.orient.core.metadata.security.OUser;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
import com.orientechnologies.orient.core.type.ODocumentWrapper;
import com.orientechnologies.orient.server.OServer;
import com.orientechnologies.orient.server.OServerMain;
import javax.persistence.Id;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;


public class Main {

    public static class GeoIp  {

          @Id
          private Object id;
          @OVersion
          private Object version;


        private String from;
        private String to;
        private Long lat;
        private Long lng;
        private String country;
        private String state;

        public void setFrom(String from) {
            this.from = from;
        }

        public void setTo(String to) {
            this.to = to;
        }

        public void setCountry(String country) {
            this.country = country;
        }

        public void setState(String state) {
            this.state = state;
        }

        public String getFrom() {
            return from;
        }

        public String getTo() {
            return to;
        }

        public String getCountry() {
            return country;
        }

        public String getState() {
            return state;
        }

        public Long getLng() {
            return lng;
        }

        public Long getLat() {
            return lat;
        }

        public void setLng(Long lng) {
            this.lng = lng;
        }

        public void setLat(Long lat) {
            this.lat = lat;
        }
    }

    public static void main(String[] args) {
        try {

            String base = "/home/marco/Scrivania/orientdb/";

            OServer server = OServerMain.create();
            server.startup(new File(base + "/file/conf.xml"));


                ODatabaseObjectTx db = new ODatabaseObjectTx("local:" + base + "/db").create();
                db.getEntityManager().registerEntityClass(GeoIp.class);
                
            try {


                for (String s : db.getClusterNames()) {
                    System.out.println("name: " + s + " - " + db.countClusterElements(s));

                }

                db.begin();


                BigFile geoip_list = new BigFile(base + "/file/data/GeoIPCountryWhois.csv");


                long time = System.currentTimeMillis();


                long c = 0;


                for (String geoip_row : geoip_list) {
                    geoip_row = geoip_row.replaceAll("\"", "");
                    String[] s = geoip_row.split("\\,");
                    GeoIp geoIp = new GeoIp();
                    geoIp.setFrom(s[0]);
                    geoIp.setTo(s[1]);
                    geoIp.setLat(new Long(s[2]));
                    geoIp.setLng(new Long(s[3]));
                    geoIp.setCountry(s[4]);
                    geoIp.setState(s[5]);
                    db.save(geoIp);
                    if(c>10000 && c % 10000 == 0)
                        System.out.println(c);
                    c++;
                }



                db.commit();
                System.out.println("tot time: " + (System.currentTimeMillis() - time));
                System.out.println("commit:" + c);


            } catch (Exception e) {
                System.out.println("e:" + e.getMessage() + e.getStackTrace().toString());
                db.rollback();
            } finally {


                for (String s : db.getClusterNames()) {
                    System.out.println("name: " + s + " - " + db.countClusterElements(s));

                }

                db.close();
            }


            server.shutdown();
        } catch (Exception ex) {
            System.out.println("ex:" + ex.getMessage()+ ex.getStackTrace().toString());
        }


    }
}

qui vengono indicate le velocità (colonna speed) in base al metodologia di salvataggio usato.


altre puntate: #1