1. 程式人生 > >Java連線HBASE資料庫,建立一個表,刪除一張表,修改表,輸出插入,修改,資料刪除,資料獲取,顯示錶資訊,過濾查詢,分頁查詢,地理hash

Java連線HBASE資料庫,建立一個表,刪除一張表,修改表,輸出插入,修改,資料刪除,資料獲取,顯示錶資訊,過濾查詢,分頁查詢,地理hash

準備工作
1、建立Java的Maven專案
建立好的目錄結構如下:
這裡寫圖片描述

另外注意junit的版本,最好不要太高,最開始筆者使用的junit4.12的,發現執行的時候會報錯。最後把Junit的版本改成4.7的之後,問題解決了。

編寫POM檔案

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation
="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <groupId>com.youx</groupId> <artifactId>hbase-demo</artifactId> <version>1.0-SNAPSHOT</version> <dependencies> <!--<dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase</artifactId> <version>1.4.2</version> </dependency>-->
<dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-client</artifactId> <version>1.4.2</version> </dependency> <dependency> <groupId>junit</groupId> <artifactId
>
junit</artifactId> <version>4.12</version> </dependency> </dependencies> </project>

建立測試工具類HbaseDemo,為了保證能夠讓HBASE執行,需要最開始寫出如下配置:

package toto.com.hbase;

import java.util.ArrayList;
import java.util.Iterator;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
import org.apache.hadoop.hbase.filter.ByteArrayComparable;
import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.FamilyFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.filter.MultipleColumnPrefixFilter;
import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Before;
import org.junit.Test;

/**
 * 程式碼說明
 *
 * @author tuzq
 * @create 2017-06-22 20:48
 */
public class HbaseDemo {
    private Configuration conf = null;
    private Connection conn = null;

    @Before
    public void init() throws Exception {
        conf = HBaseConfiguration.create();
        conn = ConnectionFactory.createConnection();
    }
}

2、建立一個表

@Test
    public void testCreate() throws Exception {
        //獲取一個表管理器
        Admin admin = conn.getAdmin();
        //表的描述器
        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("t_user_info"));

        //列族的描述器
        HColumnDescriptor hcd1 = new HColumnDescriptor("base_info");
        //BloomType.ROW 快速搜尋一個數據在哪個檔案塊中
        hcd1.setBloomFilterType(BloomType.ROW).setVersions(1,3);

        //列族的描述器
        HColumnDescriptor hcd2 = new HColumnDescriptor("extra_info");
        hcd2.setBloomFilterType(BloomType.ROW).setVersions(1,3);

        //新增列族
        htd.addFamily(hcd1).addFamily(hcd2);

        //建立表
        admin.createTable(htd);
        admin.close();
        conn.close();
    }

執行後的成功的結果狀態:
這裡寫圖片描述
進入HBASE中,檢視資料資訊
這裡寫圖片描述

3、刪除一張表

@Test
    public void testDrop() throws Exception {
        Admin admin = conn.getAdmin();
        admin.disableTable(TableName.valueOf("t_user_info2"));
        admin.deleteTable(TableName.valueOf("t_user_info2"));
        admin.close();
        conn.close();
    }

執行後的效果如下:
這裡寫圖片描述
進入HBASE中檢視效果:
這裡寫圖片描述

4.其它操作api

@Test
    public void testModify() throws Exception {
        Admin admin = conn.getAdmin();
        // 修改已有ColumnFamily
        HTableDescriptor table = admin.getTableDescriptor(TableName.valueOf("t_user_info"));
        HColumnDescriptor f2 = table.getFamily("extra_info".getBytes());
        f2.setBloomFilterType(BloomType.ROWCOL);
        // 新增新的ColumnFamily
        table.addFamily(new HColumnDescriptor("other_info"));

        admin.modifyTable(TableName.valueOf("t_user_info"), table);
        admin.close();
        conn.close();
    }

    /**
     * 插入/修改 資料
     * @throws Exception
     */
    @Test
    public void testPut() throws Exception {
        Table table = conn.getTable(TableName.valueOf("t_user_info"));

        ArrayList<Put> puts = new ArrayList<Put>();

        Put put01 = new Put(Bytes.toBytes("user001"));
        put01.addColumn(Bytes.toBytes("base_info"), Bytes.toBytes("username"), Bytes.toBytes("zhangsan"));

        Put put02 = new Put("user001".getBytes());
        put02.addColumn(Bytes.toBytes("base_info"), Bytes.toBytes("password"), Bytes.toBytes("123456"));

        Put put03 = new Put("user002".getBytes());
        put03.addColumn(Bytes.toBytes("base_info"), Bytes.toBytes("username"), Bytes.toBytes("lisi"));
        put03.addColumn(Bytes.toBytes("extra_info"), Bytes.toBytes("married"), Bytes.toBytes("false"));

        Put put04 = new Put("zhang_sh_01".getBytes());
        put04.addColumn(Bytes.toBytes("base_info"), Bytes.toBytes("username"), Bytes.toBytes("zhang01"));
        put04.addColumn(Bytes.toBytes("extra_info"), Bytes.toBytes("married"), Bytes.toBytes("false"));

        Put put05 = new Put("zhang_sh_02".getBytes());
        put05.addColumn(Bytes.toBytes("base_info"), Bytes.toBytes("username"), Bytes.toBytes("zhang02"));
        put05.addColumn(Bytes.toBytes("extra_info"), Bytes.toBytes("married"), Bytes.toBytes("false"));

        Put put06 = new Put("liu_sh_01".getBytes());
        put06.addColumn(Bytes.toBytes("base_info"), Bytes.toBytes("username"), Bytes.toBytes("liu01"));
        put06.addColumn(Bytes.toBytes("extra_info"), Bytes.toBytes("married"), Bytes.toBytes("false"));

        Put put07 = new Put("zhang_bj_01".getBytes());
        put07.addColumn(Bytes.toBytes("base_info"), Bytes.toBytes("username"), Bytes.toBytes("zhang03"));
        put07.addColumn(Bytes.toBytes("extra_info"), Bytes.toBytes("married"), Bytes.toBytes("false"));

        Put put08 = new Put("zhang_bj_01".getBytes());
        put08.addColumn(Bytes.toBytes("base_info"), Bytes.toBytes("username"), Bytes.toBytes("zhang04"));
        put08.addColumn(Bytes.toBytes("extra_info"), Bytes.toBytes("married"), Bytes.toBytes("false"));

        puts.add(put01);
        puts.add(put02);
        puts.add(put03);
        puts.add(put04);
        puts.add(put05);
        puts.add(put06);
        puts.add(put07);
        puts.add(put08);

        table.put(puts);
        table.close();
        conn.close();
    }

    @Test
    public void testDel() throws Exception {
        Table t_user_info = conn.getTable(TableName.valueOf("t_user_info"));
        Delete delete = new Delete("user001".getBytes());
        delete.addColumn("base_info".getBytes(), "password".getBytes());
        t_user_info.delete(delete);

        t_user_info.close();
        conn.close();
    }
@Test
    public void testGet() throws Exception {
        Table table = conn.getTable(TableName.valueOf("t_user_info"));

        Get get = new Get("user001".getBytes());
        Result result = table.get(get);
        CellScanner cellScanner = result.cellScanner();
        while(cellScanner.advance()) {
            Cell current = cellScanner.current();
            byte[] familyArray = current.getFamilyArray();
            byte[] qualifierArray = current.getQualifierArray();
            byte[] valueArray = current.getValueArray();

            System.out.println(new String(familyArray,current.getFamilyLength(),current.getFamilyLength()));
            System.out.print(":" + new String(qualifierArray, current.getQualifierOffset(), current.getQualifierLength()));
            System.out.print(":" + new String(qualifierArray, current.getQualifierOffset(), current.getQualifierLength()));
        }
        table.close();
        table.close();
    }

執行結果:
這裡寫圖片描述

@Test
    public void testScan() throws Exception {
        Table t_user_info = conn.getTable(TableName.valueOf("t_user_info"));
        Scan scan = new Scan();

        ResultScanner scanner = t_user_info.getScanner(scan);
        Iterator<Result> iter = scanner.iterator();
        while (iter.hasNext()) {
            Result result = iter.next();
            CellScanner cellScanner = result.cellScanner();
            while(cellScanner.advance()) {
                Cell current = cellScanner.current();
                byte[] familyArray = current.getFamilyArray();
                byte[] valueArray = current.getValueArray();
                byte[] qualifierArray = current.getQualifierArray();
                byte[] rowArray = current.getRowArray();

                System.out.println(new String(rowArray, current.getRowOffset(), current.getRowLength()));
                System.out.print(new String(familyArray, current.getFamilyOffset(), current.getFamilyLength()));
                System.out.print(":" + new String(qualifierArray, current.getQualifierOffset(), current.getQualifierLength()));
                System.out.println(" " + new String(valueArray, current.getValueOffset(), current.getValueLength()));
            }
            System.out.println("-----------------------");
        }
    }

執行結果:
這裡寫圖片描述

其它程式碼(過濾,掃描,分頁查詢):

/**
     * 下面的內容一個註釋一個註釋的放開,然後可以看到過濾後的效果
     * @throws Exception
     */
    @Test
    public void testFilter() throws Exception {
        // 針對行鍵的字首過濾器
        /*Filter pf = new PrefixFilter(Bytes.toBytes("liu"));
        testScan(pf);*/

        // 行過濾器
        /*RowFilter rf1 = new RowFilter(CompareOp.LESS, new BinaryComparator(Bytes.toBytes("user001")));
        RowFilter rf2 = new RowFilter(CompareOp.EQUAL, new SubstringComparator("00"));
        testScan(rf1);
        System.out.println("**********");
        testScan(rf2);*/

        // 針對指定一個列的value來過濾
        /*SingleColumnValueFilter scvf = new SingleColumnValueFilter("base_info".getBytes(), "password".getBytes(), CompareOp.EQUAL, "123456".getBytes());
        scvf.setFilterIfMissing(true);   // 如果指定的列缺失,則也過濾掉
        testScan(scvf);*/

        /*ByteArrayComparable comparator1 = new RegexStringComparator("^zhang");
        ByteArrayComparable comparator2 = new SubstringComparator("ang");
        SingleColumnValueFilter scvf = new SingleColumnValueFilter("base_info".getBytes(), "username".getBytes(), CompareOp.EQUAL, comparator1);
        testScan(scvf);*/

        // 針對列族名的過濾器
        /*FamilyFilter ff1 = new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("inf")));
        FamilyFilter ff2 = new FamilyFilter(CompareOp.EQUAL, new BinaryPrefixComparator(Bytes.toBytes("base")));
        testScan(ff1);*/

        // 針對列名的過濾器
        /*QualifierFilter qf = new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("password")));
        QualifierFilter qf2 = new QualifierFilter(CompareOp.EQUAL, new BinaryPrefixComparator(Bytes.toBytes("us")));
        testScan(qf);*/

        // 跟SingleColumnValueFilter結果不同,只返回符合條件的該column
        /*ColumnPrefixFilter cf = new ColumnPrefixFilter("passw".getBytes());
        testScan(cf);*/

        /*byte[][] prefixes = new byte[][] { Bytes.toBytes("username"),Bytes.toBytes("password") };
        MultipleColumnPrefixFilter mcf = new MultipleColumnPrefixFilter(prefixes);
        testScan(mcf);*/

        /*FamilyFilter ff2 = new FamilyFilter(CompareOp.EQUAL, new BinaryPrefixComparator(Bytes.toBytes("base")));
        ColumnPrefixFilter cf = new ColumnPrefixFilter("passw".getBytes());
        FilterList filterList = new FilterList(Operator.MUST_PASS_ALL);
        filterList.addFilter(ff2);
        filterList.addFilter(cf);
        testScan(filterList);*/
    }

    public void testScan(Filter filter) throws Exception {

        Table t_user_info = conn.getTable(TableName.valueOf("t_user_info"));

        Scan scan = new Scan();
        scan.setFilter(filter);
        ResultScanner scanner = t_user_info.getScanner(scan);

        Iterator<Result> iter = scanner.iterator();
        while (iter.hasNext()) {
            Result result = iter.next();
            CellScanner cellScanner = result.cellScanner();
            while (cellScanner.advance()) {
                Cell current = cellScanner.current();
                byte[] familyArray = current.getFamilyArray();
                byte[] valueArray = current.getValueArray();
                byte[] qualifierArray = current.getQualifierArray();
                byte[] rowArray = current.getRowArray();

                System.out.println(new String(rowArray, current.getRowOffset(), current.getRowLength()));
                System.out.print(new String(familyArray, current.getFamilyOffset(), current.getFamilyLength()));
                System.out.print(":" + new String(qualifierArray, current.getQualifierOffset(), current.getQualifierLength()));
                System.out.println(" " + new String(valueArray, current.getValueOffset(), current.getValueLength()));
            }
            System.out.println("-----------------------");
        }
    }

    /**
     * 分頁查詢
     *
     * @throws Exception
     */
    @Test
    public void pageScan() throws Exception{
        final byte[] POSTFIX = new byte[] { 0x00 };
        Table table = conn.getTable(TableName.valueOf("t_user_info"));
        Filter filter = new PageFilter(3);   // 一次需要獲取一頁的條數
        byte[] lastRow = null;
        int totalRows = 0;
        while (true) {
            Scan scan = new Scan();
            scan.setFilter(filter);
            if(lastRow != null){
                byte[] startRow = Bytes.add(lastRow,POSTFIX);   //設定本次查詢的起始行鍵
                scan.setStartRow(startRow);
            }
            ResultScanner scanner = table.getScanner(scan);
            int localRows = 0;
            Result result;
            while((result = scanner.next()) != null){
                System.out.println(localRows++ + ":" + result);
                totalRows ++;
                lastRow = result.getRow();
            }
            scanner.close();
            if(localRows == 0) break;
            Thread.sleep(2000);
        }
        System.out.println("total rows:" + totalRows);
    }

    public static void main(String[] args) throws Exception {
        HbaseDemo demo = new HbaseDemo();
        demo.init();
        demo.testScan();
    }

地理hash的程式碼:


import java.util.BitSet;
import java.util.HashMap;

public class Geohash {

    private static int numbits = 6 * 5;
    final static char[] digits = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'j', 'k', 'm', 'n', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z' };

    final static HashMap<Character, Integer> lookup = new HashMap<Character, Integer>();
    static {
        int i = 0;
        for (char c : digits)
            lookup.put(c, i++);
    }

    public double[] decode(String geohash) {
        StringBuilder buffer = new StringBuilder();
        for (char c : geohash.toCharArray()) {

            int i = lookup.get(c) + 32;
            buffer.append(Integer.toString(i, 2).substring(1));
        }

        BitSet lonset = new BitSet();
        BitSet latset = new BitSet();

        // even bits
        int j = 0;
        for (int i = 0; i < numbits * 2; i += 2) {
            boolean isSet = false;
            if (i < buffer.length())
                isSet = buffer.charAt(i) == '1';
            lonset.set(j++, isSet);
        }

        // odd bits
        j = 0;
        for (int i = 1; i < numbits * 2; i += 2) {
            boolean isSet = false;
            if (i < buffer.length())
                isSet = buffer.charAt(i) == '1';
            latset.set(j++, isSet);
        }
        // 中國地理座標:東經73°至東經135°,北緯4°至北緯53°
        double lon = decode(lonset, 70, 140);
        double lat = decode(latset, 0, 60);

        return new double[] { lat, lon };
    }

    private double decode(BitSet bs, double floor, double ceiling) {
        double mid = 0;
        for (int i = 0; i < bs.length(); i++) {
            mid = (floor + ceiling) / 2;
            if (bs.get(i))
                floor = mid;
            else
                ceiling = mid;
        }
        return mid;
    }

    public String encode(double lat, double lon) {
        BitSet latbits = getBits(lat, 0, 60);
        BitSet lonbits = getBits(lon, 70, 140);
        StringBuilder buffer = new StringBuilder();
        for (int i = 0; i < numbits; i++) {
            buffer.append((lonbits.get(i)) ? '1' : '0');
            buffer.append((latbits.get(i)) ? '1' : '0');
        }
        return base32(Long.parseLong(buffer.toString(), 2));
    }

    private BitSet getBits(double lat, double floor, double ceiling) {
        BitSet buffer = new BitSet(numbits);
        for (int i = 0; i < numbits; i++) {
            double mid = (floor + ceiling) / 2;
            if (lat >= mid) {
                buffer.set(i);
                floor = mid;
            } else {
                ceiling = mid;
            }
        }
        return buffer;
    }

    public static String base32(long i) {
        char[] buf = new char[65];
        int charPos = 64;
        boolean negative = (i < 0);
        if (!negative)
            i = -i;
        while (i <= -32) {
            buf[charPos--] = digits[(int) (-(i % 32))];
            i /= 32;
        }
        buf[charPos] = digits[(int) (-i)];

        if (negative)
            buf[--charPos] = '-';
        return new String(buf, charPos, (65 - charPos));
    }

    public static void main(String[] args) {
        Geohash geohash = new Geohash();
        String encode1 = geohash.encode(123.3345, 78.2247);
        String encode2 = geohash.encode(123.3446, 78.2249);
        String encode3 = geohash.encode(120.3346, 108.2249);
        System.out.println(encode1);
        System.out.println(encode2);
        System.out.println(encode3);
    }

}