hadoop（十二）hbase資料庫+叢集搭建+Api
HBASE表結構：
建表時，不需要限定表中的欄位，只需要指定若干個列族
插入資料時，列族中可以儲存任意多個列（KV，列名&列值）
要查詢某一列具體欄位的值，需要指定的座標：列名--->行健--->列族：列名--->版本
hbase的存放策略：
一張大表切分為多個Region，Region放在多個Region Server上，Region Server將Region放在HDFS裡（datanode）,通常
Region Server和datanode在一臺機器上
hbase查詢（定址機制）
略略略。
hbase叢集搭建
1.上傳hbase安裝包
2.解壓
3.配置hbase叢集，要修改3個檔案（首先zk叢集已經安裝好了）
注意：要把hadoop的hdfs-site.xml和core-site.xml 放到hbase/conf下
3.1修改hbase-env.sh
export JAVA_HOME=/usr/java/jdk1.7.0_55
//告訴hbase使用外部的zk
export HBASE_MANAGES_ZK=false
vim hbase-site.xml
<configuration>
<!-- 指定hbase在HDFS上儲存的路徑 -->
<property>
<name>hbase.rootdir</name>
<value>hdfs://ns1/hbase</value>
</property>
<!-- 指定hbase是分散式的 -->
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<!-- 指定zk的地址，多個用“,”分割 -->
<property>
<name>hbase.zookeeper.quorum</name>
<value>weekend04:2181,weekend05:2181,weekend06:2181</value>
</property>
</configuration>
vim regionservers
weekend03
weekend04
weekend05
weekend06
3.2拷貝hbase到其他節點
scp -r /weekend/hbase-0.96.2-hadoop2/ weekend02:/weekend/
scp -r /weekend/hbase-0.96.2-hadoop2/ weekend03:/weekend/
scp -r /weekend/hbase-0.96.2-hadoop2/ weekend04:/weekend/
scp -r /weekend/hbase-0.96.2-hadoop2/ weekend05:/weekend/
scp -r /weekend/hbase-0.96.2-hadoop2/ weekend06:/weekend/
4.將配置好的HBase拷貝到每一個節點並同步時間。
5.啟動所有的hbase
分別啟動zk
./zkServer.sh start
啟動hbase叢集
start-dfs.sh
啟動hbase，在主節點上執行：
start-hbase.sh
6.通過瀏覽器訪問hbase管理頁面
192.168.1.201:60010
7.為保證叢集的可靠性，要啟動多個HMaster
hbase-daemon.sh start master
hbase-shell
./hbase shell 進入客戶端
list 檢視所有的表
describe 描述表結構
put存資料
put '表名'，'行名'，'列名','值'
put '表名', '0001', 'base_info:name', 'fengjie'
put '表名', '0001', 'base_info:age', '18'
put '表名', '0001', 'extra_info:boyfriend', 'xiaoming'
get獲取資料
get '表名'，'0001'
用drop命令可以刪除表。在刪除一個表之前必須先將其禁用。
disable 'emp'
drop 'emp'
scan 'table1' 查看錶資料
hbase- Java Api
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
import org.apache.hadoop.hbase.filter.ByteArrayComparable;
import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.FamilyFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.MultipleColumnPrefixFilter;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.master.TableNamespaceManager;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Before;
import org.junit.Test;
public class HbaseDemo {
private Configuration conf = null;
@Before
public void init(){
conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", "weekend05,weekend06,weekend07");
}
@Test
public void testDrop() throws Exception{
HBaseAdmin admin = new HBaseAdmin(conf);
admin.disableTable("account");
admin.deleteTable("account");
admin.close();
}
@Test
public void testPut() throws Exception{
HTable table = new HTable(conf, "person_info");
Put p = new Put(Bytes.toBytes("person_rk_bj_zhang_000002"));
p.add("base_info".getBytes(), "name".getBytes(), "zhangwuji".getBytes());
table.put(p);
table.close();
}
@Test
public void testGet() throws Exception{
HTable table = new HTable(conf, "person_info");
Get get = new Get(Bytes.toBytes("person_rk_bj_zhang_000001"));
get.setMaxVersions(5);
Result result = table.get(get);
List<Cell> cells = result.listCells();
// result.getValue(family, qualifier); 可以從result中直接取出一個特定的value
//遍歷出result中所有的鍵值對
for(KeyValue kv : result.list()){
String family = new String(kv.getFamily());
System.out.println(family);
String qualifier = new String(kv.getQualifier());
System.out.println(qualifier);
System.out.println(new String(kv.getValue()));
}
table.close();
}
/**
* 多種過濾條件的使用方法
* @throws Exception
*/
@Test
public void testScan() throws Exception{
HTable table = new HTable(conf, "person_info".getBytes());
Scan scan = new Scan(Bytes.toBytes("person_rk_bj_zhang_000001"), Bytes.toBytes("person_rk_bj_zhang_000002"));
//字首過濾器----針對行鍵
Filter filter = new PrefixFilter(Bytes.toBytes("rk"));
//行過濾器
ByteArrayComparable rowComparator = new BinaryComparator(Bytes.toBytes("person_rk_bj_zhang_000001"));
RowFilter rf = new RowFilter(CompareOp.LESS_OR_EQUAL, rowComparator);
/**
* 假設rowkey格式為：建立日期_釋出日期_ID_TITLE
* 目標：查詢 釋出日期 為 2014-12-21 的資料
*/
rf = new RowFilter(CompareOp.EQUAL , new SubstringComparator("_2014-12-21_"));
//單值過濾器 1 完整匹配位元組陣列
new SingleColumnValueFilter("base_info".getBytes(), "name".getBytes(), CompareOp.EQUAL, "zhangsan".getBytes());
//單值過濾器2 匹配正則表示式
ByteArrayComparable comparator = new RegexStringComparator("zhang.");
new SingleColumnValueFilter("info".getBytes(), "NAME".getBytes(), CompareOp.EQUAL, comparator);
//單值過濾器2 匹配是否包含子串,大小寫不敏感
comparator = new SubstringComparator("wu");
new SingleColumnValueFilter("info".getBytes(), "NAME".getBytes(), CompareOp.EQUAL, comparator);
//鍵值對元資料過濾-----family過濾----位元組陣列完整匹配
FamilyFilter ff = new FamilyFilter(
CompareOp.EQUAL ,
new BinaryComparator(Bytes.toBytes("base_info")) //表中不存在inf列族，過濾結果為空
);
//鍵值對元資料過濾-----family過濾----位元組陣列字首匹配
ff = new FamilyFilter(
CompareOp.EQUAL ,
new BinaryPrefixComparator(Bytes.toBytes("inf")) //表中存在以inf打頭的列族info，過濾結果為該列族所有行
);
//鍵值對元資料過濾-----qualifier過濾----位元組陣列完整匹配
filter = new QualifierFilter(
CompareOp.EQUAL ,
new BinaryComparator(Bytes.toBytes("na")) //表中不存在na列，過濾結果為空
);
filter = new QualifierFilter(
CompareOp.EQUAL ,
new BinaryPrefixComparator(Bytes.toBytes("na")) //表中存在以na打頭的列name，過濾結果為所有行的該列資料
);
//基於列名(即Qualifier)字首過濾資料的ColumnPrefixFilter
filter = new ColumnPrefixFilter("na".getBytes());
//基於列名(即Qualifier)多個字首過濾資料的MultipleColumnPrefixFilter
byte[][] prefixes = new byte[][] {Bytes.toBytes("na"), Bytes.toBytes("me")};
filter = new MultipleColumnPrefixFilter(prefixes);
//為查詢設定過濾條件
scan.setFilter(filter);
scan.addFamily(Bytes.toBytes("base_info"));
ResultScanner scanner = table.getScanner(scan);
for(Result r : scanner){
/**
for(KeyValue kv : r.list()){
String family = new String(kv.getFamily());
System.out.println(family);
String qualifier = new String(kv.getQualifier());
System.out.println(qualifier);
System.out.println(new String(kv.getValue()));
}
*/
//直接從result中取到某個特定的value
byte[] value = r.getValue(Bytes.toBytes("base_info"), Bytes.toBytes("name"));
System.out.println(new String(value));
}
table.close();
}
@Test
public void testDel() throws Exception{
HTable table = new HTable(conf, "user");
Delete del = new Delete(Bytes.toBytes("rk0001"));
del.deleteColumn(Bytes.toBytes("data"), Bytes.toBytes("pic"));
table.delete(del);
table.close();
}
public static void main(String[] args) throws Exception {
Configuration conf = HBaseConfiguration.create();
// conf.set("hbase.zookeeper.quorum", "weekend05:2181,weekend06:2181,weekend07:2181");
HBaseAdmin admin = new HBaseAdmin(conf);
TableName tableName = TableName.valueOf("person_info");
HTableDescriptor td = new HTableDescriptor(tableName);
HColumnDescriptor cd = new HColumnDescriptor("base_info");
cd.setMaxVersions(10);
td.addFamily(cd);
admin.createTable(td);
admin.close();
}
}