浅谈HBase在SpringBoot项目里的应用(含HBaseUtil工具类)

<dependency>
 <groupId>org.apache.hbase</groupId>
 <artifactId>hbase-client</artifactId>
 <version>1.3.0</version>
 <exclusions>
  <exclusion>
   <groupId>org.slf4j</groupId>
   <artifactId>slf4j-log4j12</artifactId>
  </exclusion>
  <exclusion>
   <groupId>log4j</groupId>
   <artifactId>log4j</artifactId>
  </exclusion>
  <exclusion>
   <groupId>javax.servlet</groupId>
   <artifactId>servlet-api</artifactId>
  </exclusion>
 </exclusions>
</dependency>

<dependency>
 <groupId>org.apache.hadoop</groupId>
 <artifactId>hadoop-common</artifactId>
 <version>2.6.0</version>
</dependency>

<dependency>
 <groupId>org.apache.hadoop</groupId>
 <artifactId>hadoop-mapreduce-client-core</artifactId>
 <version>2.6.0</version>
</dependency>

<dependency>
 <groupId>org.apache.hadoop</groupId>
 <artifactId>hadoop-mapreduce-client-common</artifactId>
 <version>2.6.0</version>
</dependency>

<dependency>
 <groupId>org.apache.hadoop</groupId>
 <artifactId>hadoop-hdfs</artifactId>
 <version>2.6.0</version>
</dependency>

import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import java.util.Map;

/**
 * Hbase-Conf配置
 *
 * @Author: yuanj
 * @Date: 2018/10/12 10:49
 */
@Configuration
@ConfigurationProperties(prefix = HbaseConfig.CONF_PREFIX)
public class HbaseConfig {

 public static final String CONF_PREFIX = "hbase.conf";

 private Map<String,String> confMaps;

 public Map<String, String> getconfMaps() {
  return confMaps;
 }
 public void setconfMaps(Map<String, String> confMaps) {
  this.confMaps = confMaps;
 }
}

package com.moerlong.credit.core;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;

/**
 * Spring的ApplicationContext的持有者,可以用静态方法的方式获取spring容器中的bean
 */
@Component
public class SpringContextHolder implements ApplicationContextAware {

 private static ApplicationContext applicationContext;

 @Override
 public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
  SpringContextHolder.applicationContext = applicationContext;
 }

 public static ApplicationContext getApplicationContext() {
  assertApplicationContext();
  return applicationContext;
 }

 @SuppressWarnings("unchecked")
 public static <T> T getBean(String beanName) {
  assertApplicationContext();
  return (T) applicationContext.getBean(beanName);
 }

 public static <T> T getBean(Class<T> requiredType) {
  assertApplicationContext();
  return applicationContext.getBean(requiredType);
 }

 private static void assertApplicationContext() {
  if (SpringContextHolder.applicationContext == null) {
   throw new RuntimeException("applicaitonContext属性为null,请检查是否注入了SpringContextHolder!");
  }
 }

}

import com.moerlong.credit.config.HbaseConfig;
import com.moerlong.credit.core.SpringContextHolder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.client.coprocessor.AggregationClient;
import org.apache.hadoop.hbase.client.coprocessor.LongColumnInterpreter;
import org.apache.hadoop.hbase.filter.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.DependsOn;
import org.springframework.stereotype.Component;
import org.springframework.util.StopWatch;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

@DependsOn("springContextHolder") //控制依赖顺序，保证springContextHolder类在之前已经加载
@Component
public class HBaseUtils {

 private Logger logger = LoggerFactory.getLogger(this.getClass());
 
 //手动获取hbaseConfig配置类对象
 private static HbaseConfig hbaseConfig = SpringContextHolder.getBean("hbaseConfig");

 private static Configuration conf = HBaseConfiguration.create();
 private static ExecutorService pool = Executors.newScheduledThreadPool(20); //设置连接池
 private static Connection connection = null;
 private static HBaseUtils instance = null;
 private static Admin admin = null;

 private HBaseUtils(){
  if(connection == null){
   try {
    //将hbase配置类中定义的配置加载到连接池中每个连接里
    Map<String, String> confMap = hbaseConfig.getconfMaps();
    for (Map.Entry<String,String> confEntry : confMap.entrySet()) {
     conf.set(confEntry.getKey(), confEntry.getValue());
    }
    connection = ConnectionFactory.createConnection(conf, pool);
    admin = connection.getAdmin();
   } catch (IOException e) {
    logger.error("HbaseUtils实例初始化失败！错误信息为：" + e.getMessage(), e);
   }
  }
 } 
 //简单单例方法，如果autowired自动注入就不需要此方法
 public static synchronized HBaseUtils getInstance(){
  if(instance == null){
   instance = new HBaseUtils();
  }
  return instance;
 }

 /**
  * 创建表
  *
  * @param tableName   表名
  * @param columnFamily  列族（数组）
  */
 public void createTable(String tableName, String[] columnFamily) throws IOException{
  TableName name = TableName.valueOf(tableName);
  //如果存在则删除
  if (admin.tableExists(name)) {
   admin.disableTable(name);
   admin.deleteTable(name);
   logger.error("create htable error! this table {} already exists!", name);
  } else {
   HTableDescriptor desc = new HTableDescriptor(name);
   for (String cf : columnFamily) {
    desc.addFamily(new HColumnDescriptor(cf));
   }
   admin.createTable(desc);
  }
 }

 /**
  * 插入记录（单行单列族-多列多值）
  *
  * @param tableName   表名
  * @param row    行名
  * @param columnFamilys  列族名
  * @param columns   列名（数组）
  * @param values   值（数组）（且需要和列一一对应）
  */
 public void insertRecords(String tableName, String row, String columnFamilys, String[] columns, String[] values) throws IOException {
  TableName name = TableName.valueOf(tableName);
  Table table = connection.getTable(name);
  Put put = new Put(Bytes.toBytes(row));
  for (int i = 0; i < columns.length; i++) {
   put.addColumn(Bytes.toBytes(columnFamilys), Bytes.toBytes(columns[i]), Bytes.toBytes(values[i]));
   table.put(put);
  }
 }

 /**
  * 插入记录（单行单列族-单列单值）
  *
  * @param tableName   表名
  * @param row    行名
  * @param columnFamily  列族名
  * @param column   列名
  * @param value    值
  */
 public void insertOneRecord(String tableName, String row, String columnFamily, String column, String value) throws IOException {
  TableName name = TableName.valueOf(tableName);
  Table table = connection.getTable(name);
  Put put = new Put(Bytes.toBytes(row));
  put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column), Bytes.toBytes(value));
  table.put(put);
 }

 /**
  * 删除一行记录
  *
  * @param tablename   表名
  * @param rowkey   行名
  */
 public void deleteRow(String tablename, String rowkey) throws IOException {
  TableName name = TableName.valueOf(tablename);
  Table table = connection.getTable(name);
  Delete d = new Delete(rowkey.getBytes());
  table.delete(d);
 }

 /**
  * 删除单行单列族记录
  * @param tablename   表名
  * @param rowkey   行名
  * @param columnFamily  列族名
  */
 public void deleteColumnFamily(String tablename, String rowkey, String columnFamily) throws IOException {
  TableName name = TableName.valueOf(tablename);
  Table table = connection.getTable(name);
  Delete d = new Delete(rowkey.getBytes()).deleteFamily(Bytes.toBytes(columnFamily));
  table.delete(d);
 }

 /**
  * 删除单行单列族单列记录
  *
  * @param tablename   表名
  * @param rowkey   行名
  * @param columnFamily  列族名
  * @param column   列名
  */
 public void deleteColumn(String tablename, String rowkey, String columnFamily, String column) throws IOException {
  TableName name = TableName.valueOf(tablename);
  Table table = connection.getTable(name);
  Delete d = new Delete(rowkey.getBytes()).deleteColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column));
  table.delete(d);
 }

 /**
  * 查找一行记录
  *
  * @param tablename   表名
  * @param rowKey   行名
  */
 public static String selectRow(String tablename, String rowKey) throws IOException {
  String record = "";
  TableName name=TableName.valueOf(tablename);
  Table table = connection.getTable(name);
  Get g = new Get(rowKey.getBytes());
  Result rs = table.get(g);
  NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = rs.getMap();
  for (Cell cell : rs.rawCells()) {
   StringBuffer stringBuffer = new StringBuffer().append(Bytes.toString(cell.getRow())).append("\t")
     .append(Bytes.toString(cell.getFamily())).append("\t")
     .append(Bytes.toString(cell.getQualifier())).append("\t")
     .append(Bytes.toString(cell.getValue())).append("\n");
   String str = stringBuffer.toString();
   record += str;
  }
  return record;
 }

 /**
  * 查找单行单列族单列记录
  *
  * @param tablename   表名
  * @param rowKey   行名
  * @param columnFamily  列族名
  * @param column   列名
  * @return
  */
 public static String selectValue(String tablename, String rowKey, String columnFamily, String column) throws IOException {
  TableName name=TableName.valueOf(tablename);
  Table table = connection.getTable(name);
  Get g = new Get(rowKey.getBytes());
  g.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column));
  Result rs = table.get(g);
  return Bytes.toString(rs.value());
 }

 /**
  * 查询表中所有行（Scan方式）
  *
  * @param tablename
  * @return
  */
 public String scanAllRecord(String tablename) throws IOException {
  String record = "";
  TableName name=TableName.valueOf(tablename);
  Table table = connection.getTable(name);
  Scan scan = new Scan();
  ResultScanner scanner = table.getScanner(scan);
  try {
   for(Result result : scanner){
    for (Cell cell : result.rawCells()) {
     StringBuffer stringBuffer = new StringBuffer().append(Bytes.toString(cell.getRow())).append("\t")
       .append(Bytes.toString(cell.getFamily())).append("\t")
       .append(Bytes.toString(cell.getQualifier())).append("\t")
       .append(Bytes.toString(cell.getValue())).append("\n");
     String str = stringBuffer.toString();
     record += str;
    }
   }
  } finally {
   if (scanner != null) {
    scanner.close();
   }
  }
  return record;
 }

 /**
  * 根据rowkey关键字查询报告记录
  *
  * @param tablename
  * @param rowKeyword
  * @return
  */
 public List scanReportDataByRowKeyword(String tablename, String rowKeyword) throws IOException {
  ArrayList<> list = new ArrayList<>();

  Table table = connection.getTable(TableName.valueOf(tablename));
  Scan scan = new Scan();
 
 //添加行键过滤器，根据关键字匹配
  RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator(rowKeyword));
  scan.setFilter(rowFilter);

  ResultScanner scanner = table.getScanner(scan);
  try {
   for (Result result : scanner) {
    //TODO 此处根据业务来自定义实现
    list.add(null);
   }
  } finally {
   if (scanner != null) {
    scanner.close();
   }
  }  
  return list;
 }

 /**
  * 根据rowkey关键字和时间戳范围查询报告记录
  *
  * @param tablename
  * @param rowKeyword
  * @return
  */
 public List scanReportDataByRowKeywordTimestamp(String tablename, String rowKeyword, Long minStamp, Long maxStamp) throws IOException {
  ArrayList<> list = new ArrayList<>();

  Table table = connection.getTable(TableName.valueOf(tablename));
  Scan scan = new Scan();
  //添加scan的时间范围
  scan.setTimeRange(minStamp, maxStamp);

  RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator(rowKeyword));
  scan.setFilter(rowFilter);

  ResultScanner scanner = table.getScanner(scan);
  try {
   for (Result result : scanner) {
    //TODO 此处根据业务来自定义实现
    list.add(null);
   }
  } finally {
   if (scanner != null) {
    scanner.close();
   }
  }  
  return list;
 }

 /**
  * 删除表操作
  *
  * @param tablename
  */
 public void deleteTable(String tablename) throws IOException {
  TableName name=TableName.valueOf(tablename);
  if(admin.tableExists(name)) {
   admin.disableTable(name);
   admin.deleteTable(name);
  }
 }

 /**
  * 利用协处理器进行全表count统计
  *
  * @param tablename
  */
 public Long countRowsWithCoprocessor(String tablename) throws Throwable {
  TableName name=TableName.valueOf(tablename);
  HTableDescriptor descriptor = admin.getTableDescriptor(name);

  String coprocessorClass = "org.apache.hadoop.hbase.coprocessor.AggregateImplementation";
  if (! descriptor.hasCoprocessor(coprocessorClass)) {
   admin.disableTable(name);
   descriptor.addCoprocessor(coprocessorClass);
   admin.modifyTable(name, descriptor);
   admin.enableTable(name);
  }

  //计时
  StopWatch stopWatch = new StopWatch();
  stopWatch.start();

  Scan scan = new Scan();
  AggregationClient aggregationClient = new AggregationClient(conf);

  Long count = aggregationClient.rowCount(name, new LongColumnInterpreter(), scan);

  stopWatch.stop();
  System.out.println("RowCount：" + count + "，全表count统计耗时：" + stopWatch.getTotalTimeMillis());

  return count;
 }
}

    <dependency>
      <groupId>org.apache.hbase</groupId>
      <artifactId>hbase-shaded-client</artifactId>
      <version>1.2.6</version>
    </dependency>
<!---->

import net.cc.commons.exception.CCRuntimeException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Scope;

import java.io.IOException;
import java.util.function.Supplier;

/**
* @Author wangqiubao
* @Date 2019/9/24 15:28
* @Description
**/
@Configuration
public class UcareHbaseConfiguration {
  /**
   * 读取HBase的zookeeper地址
   */
  @Value("${hbase.zookeeper.quorum}")
  private String quorum;

  /**
   * 配置HBase连接参数
   *
   * @return
   */
  @Bean
  public org.apache.hadoop.conf.Configuration hbaseConfig() {
    org.apache.hadoop.conf.Configuration config = HBaseConfiguration.create();
    config.set(HConstants.ZOOKEEPER_QUORUM, quorum);
    return config;
  }
  //每次调用get方法就会创建一个Connection
  @Bean
  public Supplier<Connection> hbaseConnSupplier() {
    return () -> {
      try {
        return hbaseConnection();
      } catch (IOException e) {
        throw new CCRuntimeException(e);
      }
    };
  }

  @Bean
  //@Scope标明模式,默认单例模式. prototype多例模式
  //若是在其他类中直接@Autowired引入的,多例就无效了,因为那个类在初始化的时候,已经创建了创建了这个bean了,之后调用的时候,不会重新创建,若是想要实现多例,就要每次调用的时候,手动获取bean
  @Scope(value = "prototype")
  public Connection hbaseConnection() throws IOException {
    return ConnectionFactory.createConnection(hbaseConfig());
  }
}

  /**
   * 内部已实现线程安全的连接池
   */
  @Autowired
  private Connection hbaseConnection;

public void aaaa() throws IOException {
  try (Table table = hbaseConnection.getTable(TableName.valueOf("表名"))) {//获取表连接
    //配置一条数据
    // 行键
    Put put = new Put(Bytes.toBytes("key主键"));
    put.addColumn(Bytes.toBytes("列族"), Bytes.toBytes("列"), Bytes.toBytes("值"));
    .....//每个有数据的列都要一个addColumn
    //put插入数据
    table.put(put);
  }
}

try (Table table = hbaseConnection.getTable(TableName.valueOf("表名"))) {
  Result result = table.get(new Get(asRowKey(date, acid)));
  if (result == null) return null;

  // 列名为starttime，最后一条就是该航班最新的航迹
  Cell latestCell = Iterables.getLast(result.listCells());
  return AdsbTrackProto.AdsbTrack.parseFrom(CellUtil.cloneValue(latestCell));
}